From 90799497d35a944e2268de274de536f37b3c46c7 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 3 May 2023 16:19:46 +0300 Subject: [PATCH 001/233] [#114] Add remove-node IR control command Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/modules/control/ir.go | 2 + .../modules/control/ir_remove_node.go | 58 ++++++++++++++++++ .../client/netmap/{add_peer.go => peer.go} | 16 +++++ pkg/services/control/ir/rpc.go | 9 +++ pkg/services/control/ir/server/calls.go | 42 +++++++++++++ pkg/services/control/ir/service.go | 12 ++++ pkg/services/control/ir/service.pb.go | Bin 23887 -> 33564 bytes pkg/services/control/ir/service.proto | 18 ++++++ pkg/services/control/ir/service_frostfs.pb.go | Bin 10125 -> 15200 bytes pkg/services/control/ir/service_grpc.pb.go | Bin 5434 -> 7044 bytes 10 files changed, 157 insertions(+) create mode 100644 cmd/frostfs-cli/modules/control/ir_remove_node.go rename pkg/morph/client/netmap/{add_peer.go => peer.go} (70%) diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index e89dda076..ae3f8502c 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -10,6 +10,8 @@ var irCmd = &cobra.Command{ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) + irCmd.AddCommand(removeNodeCmd) initControlIRTickEpochCmd() + initControlIRRemoveNodeCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_remove_node.go b/cmd/frostfs-cli/modules/control/ir_remove_node.go new file mode 100644 index 000000000..f5b968b7f --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_remove_node.go @@ -0,0 +1,58 @@ +package control + +import ( + "encoding/hex" + "errors" + + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var removeNodeCmd = &cobra.Command{ + Use: "remove-node", + Short: "Forces a node removal from netmap", + Long: "Forces a node removal from netmap via a notary request. It should be executed on other IR nodes as well.", + Run: removeNode, +} + +func initControlIRRemoveNodeCmd() { + initControlFlags(removeNodeCmd) + + flags := removeNodeCmd.Flags() + flags.String("node", "", "Node public key as a hex string") + _ = removeNodeCmd.MarkFlagRequired("node") +} + +func removeNode(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + c := getClient(cmd, pk) + + nodeKeyStr, _ := cmd.Flags().GetString("node") + if len(nodeKeyStr) == 0 { + commonCmd.ExitOnErr(cmd, "parsing node public key: ", errors.New("key cannot be empty")) + } + nodeKey, err := hex.DecodeString(nodeKeyStr) + commonCmd.ExitOnErr(cmd, "can't decode node public key: %w", err) + + req := new(ircontrol.RemoveNodeRequest) + req.SetBody(&ircontrol.RemoveNodeRequest_Body{ + Key: nodeKey, + }) + + commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) + + var resp *ircontrol.RemoveNodeResponse + err = c.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.RemoveNode(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Node removed") +} diff --git a/pkg/morph/client/netmap/add_peer.go b/pkg/morph/client/netmap/peer.go similarity index 70% rename from pkg/morph/client/netmap/add_peer.go rename to pkg/morph/client/netmap/peer.go index dc6c25540..7ceaa0250 100644 --- a/pkg/morph/client/netmap/add_peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -41,3 +41,19 @@ func (c *Client) AddPeer(p AddPeerPrm) error { } return nil } + +// ForceRemovePeer marks the given peer as offline via a notary control transaction. +func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo) error { + if !c.client.WithNotary() { + return fmt.Errorf("peer can be forcefully removed only in notary environment") + } + + prm := UpdatePeerPrm{} + prm.SetKey(nodeInfo.PublicKey()) + prm.SetControlTX(true) + + if err := c.UpdatePeerState(prm); err != nil { + return fmt.Errorf("updating peer state: %v", err) + } + return nil +} diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index 6b2234954..1b635c149 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -11,6 +11,7 @@ const serviceName = "ircontrol.ControlService" const ( rpcHealthCheck = "HealthCheck" rpcTickEpoch = "TickEpoch" + rpcRemoveNode = "RemoveNode" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -31,6 +32,14 @@ func TickEpoch( return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...) } +func RemoveNode( + cli *client.Client, + req *RemoveNodeRequest, + opts ...client.CallOption, +) (*RemoveNodeResponse, error) { + return sendUnary[RemoveNodeRequest, RemoveNodeResponse](cli, rpcRemoveNode, req, opts...) +} + func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { var resp O wResp := &responseWrapper[*O]{ diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 56e2e3f79..680d1e606 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -1,9 +1,11 @@ package control import ( + "bytes" "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -57,3 +59,43 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return resp, nil } + +// RemoveNode forces a node removal. +// +// If request is not signed with a key from white list, permission error returns. +func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + resp := new(control.RemoveNodeResponse) + resp.SetBody(new(control.RemoveNodeResponse_Body)) + + nm, err := s.netmapClient.NetMap() + if err != nil { + return nil, fmt.Errorf("getting netmap: %w", err) + } + var nodeInfo netmap.NodeInfo + for _, info := range nm.Nodes() { + if bytes.Equal(info.PublicKey(), req.GetBody().GetKey()) { + nodeInfo = info + break + } + } + if len(nodeInfo.PublicKey()) == 0 { + return nil, status.Error(codes.NotFound, "no such node") + } + if nodeInfo.IsOffline() { + return nil, status.Error(codes.FailedPrecondition, "node is already offline") + } + + if err := s.netmapClient.ForceRemovePeer(nodeInfo); err != nil { + return nil, fmt.Errorf("forcing node removal: %w", err) + } + + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go index 1aaec2c87..b2db2b43a 100644 --- a/pkg/services/control/ir/service.go +++ b/pkg/services/control/ir/service.go @@ -32,3 +32,15 @@ func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { x.Body = v } } + +func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) { + if x != nil { + x.Body = v + } +} + +func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) { + if x != nil { + x.Body = v + } +} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 84acdfc82dba1a3a8fb147772098614fea461ea8..bec74a3beaecfdcc8792ed3dc368d640e5f72eac 100644 GIT binary patch delta 3014 zcma)8YfM{Z7|tnBdKpp{N}=3N%L)_GjW46?oIoPy$zmiiG0Pz8KR5io(=tw>sDDoTzVG)w z@B2L8d%p86TxZ_Dpt|yfIdgJ)Tw*(=(a1@uBQhxAb=7%%kg3-xTy-fOcwn4Ty3QK% zl;&Hh5=;e>($$+5Qkr+OY*}lNozk%r(!?bC%Legg;i7V3Ae~d1r_#r3-7~rwoo?vF z*Z^BT#a3*9v9rz6WK(2ty48`>~oYsDYN{ng4+6XTJw z3CT{hnvf<*rBga>#S~ko)b2~&5PB^MGF;NPD|`6RRH`%=Oe1(;P2g|OJS^P-)|!Ea6dj%vyoYbf7Nes@ z%PM?QK^L0wv*jUbgOi4lKh#OUV`+y8-+7EGofbcF`Ka@o zQS7at=6Sz>GhVB06^7#f7fwDZn=;d!5@`mn2zGEj5g&;)c-QBokoOwu(IZ+gBSW#dV`S~n!KCcDheHEgGW z*Q!e~(BP$iYXYo>mv@OD#7bQ)^*w7S!E8en!huX`vP~97BC%%N3gpiT+YkyEP~_Io zn5D6VVX?bWhCSCp);>h&YsoQ_93goQ zF1FM#BIaA#m|8q(E{46;f!|s)@OMa$(N>-u^Gl&3+-TLjyoMRS1;mcN^0DC&yMtwE zV?39b8VQzY@#DBhio1teC+y= zh|h1PX1XYDq8-{H`bA(!JG8dpyA=?=5v9!73x@?DJplC33ecJrc- z$JMY2z2S7!gflYY{dbcw;6=2A?YcD&GQ)Y83mezFZiS0r2gp73kiP0AY70=;xXTpUw aut72VUl!u$gY%|xKb2+e z5w@G|;g@Y)9d|zB@TES1jM0GWav|oW96Idlu$p;PyNhw!VOO8ZZwYKo%*5lnhb84O z*UU^teDXe5f1a=#jc!S>x<=y)J6)nstb;Am3R}BHOM&8H;3ft7XpK&ksB(>py?E1A zhs9G0-UY%q9Mcpen2lU$M`=eDN3YF68@2)lY;`*d6jn<`Tk$)j!&0L7uC8q(> diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5862e8fbd..d647db0df 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -12,6 +12,8 @@ service ControlService { rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); // Forces a new epoch to be signaled by the IR node with high probability. rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); + // Forces a node removal to be signaled by the IR node with high probability. + rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse); } // Health check request. @@ -57,3 +59,19 @@ message TickEpochResponse { Body body = 1; Signature signature = 2; } + +message RemoveNodeRequest { + message Body{ + bytes key = 1; + } + + Body body = 1; + Signature signature = 2; +} + +message RemoveNodeResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index d480f0b50dd02c8435a4ad406cc546da526d4696..c93253105317c9a7bb476e9a2da0266d8e6b4895 100644 GIT binary patch delta 294 zcmeD6e^9oeSABAyNGDfNYHog6s$YIe>SP5`&B+0Ld_qo@C8@>1nN_J8hB^urdfus( zo7eG8VMJ9bGWida+$5=5R

HJyO!lK!uY(N{LL?W^P4NyLmUW7!xx84(k+T{^SCg zyMhQ;1f>?1rWTh>zOR;!q-pY5*<;Au1O?H}=8Vfw+?1ej5=qHsZ9Z{iYc~4|Ohx8T T{-AmfN%`dKYWc|A01-O?(F$y8 delta 7 OcmaD**6Y8aR~-NlIRlpf diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 700d340ca8705a58a9a71d5a146e192633a02397..6ba214da09c0a82a003442b5b4097c03ebd6c8bc 100644 GIT binary patch delta 480 zcmdm`)ndM(lSPtKUths3zbHAiSRqj%FFz$!p(r&szbr9lvNWeTR}hGu>IW2_oX;wS z%)P*-g2dfi!Q#w3*_BluNesV^$$!~U6u)J)L(;oBhuwe)zuRmOZqrE4&nro-DA99< z&^iiQ2+5$-!qU{@5={jSB*D$yoSsaRIk`0{GG+2SE^lN<@8|x*HrYf#6iICILEh=e z++@BQWbP}zOk{2xe}w|VK)6*|D^DAYr7dc9B*)8sA@VPuWIB1*{Iiz1l-rY)*R delta 36 scmZoM-=(#olV!6Ts}b|&&72PzHyiR;Fm1Nrf6lgfjf5K0WNqn40P?5|LjV8( -- 2.45.2 From a181c9e434636aff4d75f12fcf7b5448314ea646 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 May 2023 17:16:15 +0300 Subject: [PATCH 002/233] [#332] gc: Add additional logging Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 6 ++++++ pkg/local_object_storage/shard/gc.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9b6e03499..69a3b229a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -478,4 +478,10 @@ const ( FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" + ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" + ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" + ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" + ShardGCRemoveGarbageStarted = "garbage remove started" + ShardGCRemoveGarbageCompleted = "garbage remove completed" ) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 86995cd06..82876e675 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -204,6 +204,9 @@ func (s *Shard) removeGarbage() { return } + s.log.Debug(logs.ShardGCRemoveGarbageStarted) + defer s.log.Debug(logs.ShardGCRemoveGarbageCompleted) + buf := make([]oid.Address, 0, s.rmBatchSize) var iterPrm meta.GarbageIterationPrm @@ -259,6 +262,9 @@ func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { } func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { + s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) @@ -344,6 +350,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { log := s.log.With(zap.Uint64("epoch", epoch)) log.Debug(logs.ShardStartedExpiredTombstonesHandling) + defer log.Debug(logs.ShardFinishedExpiredTombstonesHandling) const tssDeleteBatch = 50 tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) @@ -400,11 +407,12 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { tss = tss[:0] tssExp = tssExp[:0] } - - log.Debug(logs.ShardFinishedExpiredTombstonesHandling) } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { + s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) -- 2.45.2 From 800eb5e983fba1b9dc4e5048df1f71229486c5bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Apr 2023 15:45:57 +0300 Subject: [PATCH 003/233] [#125] ir: Reconfigure pprof and metrics on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-ir/config.go | 2 + cmd/frostfs-ir/httpcomponent.go | 80 ++++++++++++++++++++++++++++ cmd/frostfs-ir/main.go | 94 +++++++++------------------------ cmd/frostfs-ir/metrics.go | 21 ++++++++ cmd/frostfs-ir/pprof.go | 21 ++++++++ pkg/innerring/initialization.go | 9 ---- pkg/innerring/innerring.go | 2 +- pkg/metrics/innerring.go | 4 +- 9 files changed, 154 insertions(+), 80 deletions(-) create mode 100644 cmd/frostfs-ir/httpcomponent.go create mode 100644 cmd/frostfs-ir/metrics.go create mode 100644 cmd/frostfs-ir/pprof.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed434da5..58fe9c3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) +- Reload pprof and metrics on SIGHUP for ir (#125) ### Changed ### Fixed diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c4787f965..2e2aa1613 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -60,6 +60,8 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + pprofCmp.reload() + metricsCmp.reload() log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go new file mode 100644 index 000000000..d3cda8930 --- /dev/null +++ b/cmd/frostfs-ir/httpcomponent.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "net/http" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" +) + +type httpComponent struct { + srv *httputil.Server + address string + addressKey string + name string + handler http.Handler + shutdownDur time.Duration + shutdownTimeoutKey string + enabled bool + enabledKey string +} + +func (c *httpComponent) init() { + log.Info(fmt.Sprintf("init %s", c.name)) + c.enabled = cfg.GetBool(c.enabledKey) + c.address = cfg.GetString(c.addressKey) + c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey) + + if c.enabled { + c.srv = httputil.New( + httputil.HTTPSrvPrm{ + Address: c.address, + Handler: c.handler, + }, + httputil.WithShutdownTimeout(c.shutdownDur), + ) + } else { + log.Info(fmt.Sprintf("%s is disabled, skip", c.name)) + c.srv = nil + } +} + +func (c *httpComponent) start() { + if c.srv != nil { + log.Info(fmt.Sprintf("start %s", c.name)) + wg.Add(1) + go func() { + exitErr(c.srv.Serve()) + wg.Done() + }() + } +} + +func (c *httpComponent) shutdown() error { + if c.srv != nil { + log.Info(fmt.Sprintf("shutdown %s", c.name)) + return c.srv.Shutdown() + } + return nil +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + enabled := cfg.GetBool(c.enabledKey) + address := cfg.GetString(c.addressKey) + dur := cfg.GetDuration(c.shutdownTimeoutKey) + if enabled != c.enabled || address != c.address || dur != c.shutdownDur { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } else { + c.init() + c.start() + } + } +} diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 932b90404..84dff9101 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -4,16 +4,13 @@ import ( "context" "flag" "fmt" - "net/http" "os" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -29,15 +26,16 @@ const ( ) var ( - wg = new(sync.WaitGroup) - intErr = make(chan error) // internal inner ring errors - logPrm = new(logger.Prm) - innerRing *innerring.Server - httpServers []*httputil.Server - log *logger.Logger - cfg *viper.Viper - configFile *string - configDir *string + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + pprofCmp *httpComponent + metricsCmp *httpComponent + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -73,19 +71,17 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) - initHTTPServers(cfg) + pprofCmp = newPprofComponent() + pprofCmp.init() + + metricsCmp = newMetricsComponent() + metricsCmp.init() innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) - // start HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - exitErr(srv.Serve()) - wg.Done() - }(srv) - } + pprofCmp.start() + metricsCmp.start() // start inner ring err = innerRing.Start(ctx, intErr) @@ -103,54 +99,16 @@ func main() { log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper) { - items := []struct { - cfgPrefix string - handler func() http.Handler - }{ - {"pprof", httputil.Handler}, - {"prometheus", promhttp.Handler}, +func shutdown() { + innerRing.Stop() + if err := metricsCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) } - - httpServers = make([]*httputil.Server, 0, len(items)) - - for _, item := range items { - if !cfg.GetBool(item.cfgPrefix + ".enabled") { - log.Info(item.cfgPrefix + " is disabled, skip") - continue - } - - addr := cfg.GetString(item.cfgPrefix + ".address") - - var prm httputil.HTTPSrvPrm - - prm.Address = addr - prm.Handler = item.handler() - - httpServers = append(httpServers, - httputil.New(prm, - httputil.WithShutdownTimeout( - cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"), - ), - ), + if err := pprofCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), ) } } - -func shutdown() { - innerRing.Stop() - - // shut down HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - wg.Done() - }(srv) - } -} diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go new file mode 100644 index 000000000..debeb7d60 --- /dev/null +++ b/cmd/frostfs-ir/metrics.go @@ -0,0 +1,21 @@ +package main + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +const ( + prometheusEnabledKey = "prometheus.enabled" + prometheusAddressKey = "prometheus.address" + prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout" +) + +func newMetricsComponent() *httpComponent { + return &httpComponent{ + name: "prometheus", + enabledKey: prometheusEnabledKey, + addressKey: prometheusAddressKey, + shutdownTimeoutKey: prometheusShutdownTimeoutKey, + handler: metrics.Handler(), + } +} diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go new file mode 100644 index 000000000..0d48c11de --- /dev/null +++ b/cmd/frostfs-ir/pprof.go @@ -0,0 +1,21 @@ +package main + +import ( + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" +) + +const ( + pprofEnabledKey = "pprof.enabled" + pprofAddressKey = "pprof.address" + pprofShutdownTimeoutKey = "pprof.shutdown_timeout" +) + +func newPprofComponent() *httpComponent { + return &httpComponent{ + name: "pprof", + enabledKey: pprofEnabledKey, + addressKey: pprofAddressKey, + shutdownTimeoutKey: pprofShutdownTimeoutKey, + handler: httputil.Handler(), + } +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 84d08c4c6..1dc1d40ea 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -16,7 +16,6 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error { s.key = acc.PrivateKey() return nil } - -func (s *Server) initMetrics(cfg *viper.Viper) { - if cfg.GetString("prometheus.address") == "" { - return - } - m := metrics.NewInnerRingMetrics() - s.metrics = &m -} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index deb546f08..9119ff201 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initMetrics(cfg) + server.metrics = metrics.NewInnerRingMetrics() return server, nil } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 05b76f9c9..79db424b8 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct { } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. -func NewInnerRingMetrics() InnerRingServiceMetrics { +func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( epoch = newGauge(prometheus.GaugeOpts{ Namespace: namespace, @@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { mustRegister(epoch) mustRegister(health) - return InnerRingServiceMetrics{ + return &InnerRingServiceMetrics{ epoch: epoch, health: health, } -- 2.45.2 From a5f118a987c6df52a3dbda19c0a0f7fe151516eb Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 17 Apr 2023 19:00:54 +0300 Subject: [PATCH 004/233] [#337] subscriber: Drop unused UnsubscribeForNotification It's not really needed, closing the connection works fine when exiting and normally the app doesn't need to unsubscribe at all. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 - pkg/morph/subscriber/subscriber.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 69a3b229a..936042d2d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -195,7 +195,6 @@ const ( EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreNilBlockHandler = "ignore nil block handler" // Warn in ../node/pkg/morph/event/listener.go - SubscriberUnsubscribeForNotification = "unsubscribe for notification" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" // Warn in ../node/pkg/morph/subscriber/subscriber.go SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" // Debug in ../node/pkg/morph/subscriber/subscriber.go diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index a2e1c32eb..2d478c9c3 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -27,7 +27,6 @@ type ( // Subscriber is an interface of the NotificationEvent listener. Subscriber interface { SubscribeForNotification(...util.Uint160) error - UnsubscribeForNotification() BlockNotifications() error SubscribeForNotaryRequests(mainTXSigner util.Uint160) error @@ -97,14 +96,6 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { return nil } -func (s *subscriber) UnsubscribeForNotification() { - err := s.client.UnsubscribeAll() - if err != nil { - s.log.Error(logs.SubscriberUnsubscribeForNotification, - zap.Error(err)) - } -} - func (s *subscriber) Close() { s.client.Close() } -- 2.45.2 From 35fdf6f315dfc58c1bfcc8855aea9a08cc2ba461 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 17:05:24 +0300 Subject: [PATCH 005/233] [#337] morph: Move subscription logic to subscriber Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 26 +-- pkg/morph/client/constructor.go | 27 +-- pkg/morph/client/multi.go | 157 +++------------- pkg/morph/client/notifications.go | 280 ++--------------------------- pkg/morph/subscriber/subscriber.go | 275 ++++++++++++++++++++-------- 5 files changed, 258 insertions(+), 507 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 1c33fa5e0..832315018 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -69,9 +69,6 @@ type Client struct { // on every normal call. switchLock *sync.RWMutex - notifications chan rpcclient.Notification - subsInfo // protected with switchLock - // channel for internal stop closeChan chan struct{} @@ -566,26 +563,11 @@ func (c *Client) IsValidScript(script []byte, signers []transaction.Signer) (val // NotificationChannel returns channel than receives subscribed // notification from the connected RPC node. -// Channel is closed when connection to the RPC node has been -// lost without the possibility of recovery. +// Channel is closed when connection to the RPC node is lost. func (c *Client) NotificationChannel() <-chan rpcclient.Notification { - return c.notifications -} - -// inactiveMode switches Client to an inactive mode: -// - notification channel is closed; -// - all the new RPC request would return ErrConnectionLost; -// - inactiveModeCb is called if not nil. -func (c *Client) inactiveMode() { - c.switchLock.Lock() - defer c.switchLock.Unlock() - - close(c.notifications) - c.inactive = true - - if c.cfg.inactiveModeCb != nil { - c.cfg.inactiveModeCb() - } + c.switchLock.RLock() + defer c.switchLock.RUnlock() + return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } func (c *Client) setActor(act *actor.Actor) { diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 4232b349d..1f2a1eb8d 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -10,11 +10,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/util" @@ -108,21 +105,13 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), - logger: cfg.logger, - acc: acc, - accAddr: accAddr, - cfg: *cfg, - switchLock: &sync.RWMutex{}, - notifications: make(chan rpcclient.Notification), - subsInfo: subsInfo{ - blockRcv: make(chan *block.Block), - notificationRcv: make(chan *state.ContainedNotificationEvent), - notaryReqRcv: make(chan *result.NotaryRequestEvent), - subscribedEvents: make(map[util.Uint160]string), - subscribedNotaryEvents: make(map[util.Uint160]string), - }, - closeChan: make(chan struct{}), + cache: newClientCache(), + logger: cfg.logger, + acc: acc, + accAddr: accAddr, + cfg: *cfg, + switchLock: &sync.RWMutex{}, + closeChan: make(chan struct{}), } cli.endpoints.init(cfg.endpoints) @@ -162,7 +151,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli.setActor(act) - go cli.notificationLoop(ctx) + go cli.closeWaiter(ctx) return cli, nil } diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index fab90b446..e006ca69a 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -6,11 +6,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "go.uber.org/zap" ) @@ -34,7 +29,8 @@ func (e *endpoints) init(ee []Endpoint) { e.list = ee } -func (c *Client) switchRPC(ctx context.Context) bool { +// SwitchRPC performs reconnection and returns true if it was successful. +func (c *Client) SwitchRPC(ctx context.Context) bool { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -58,20 +54,8 @@ func (c *Client) switchRPC(ctx context.Context) bool { c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, zap.String("endpoint", newEndpoint)) - subs, ok := c.restoreSubscriptions(ctx, cli, newEndpoint, false) - if !ok { - // new WS client does not allow - // restoring subscription, client - // could not work correctly => - // closing connection to RPC node - // to switch to another one - cli.Close() - continue - } - c.client = cli c.setActor(act) - c.subsInfo = subs if c.cfg.switchInterval != 0 && !c.switchIsActive.Load() && c.endpoints.list[c.endpoints.curr].Priority != c.endpoints.list[0].Priority { @@ -82,97 +66,21 @@ func (c *Client) switchRPC(ctx context.Context) bool { return true } + c.inactive = true + + if c.cfg.inactiveModeCb != nil { + c.cfg.inactiveModeCb() + } return false } -func (c *Client) notificationLoop(ctx context.Context) { - var e any - var ok bool - - for { - c.switchLock.RLock() - bChan := c.blockRcv - nChan := c.notificationRcv - nrChan := c.notaryReqRcv - c.switchLock.RUnlock() - - select { - case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() - - return - case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() - - return - case e, ok = <-bChan: - case e, ok = <-nChan: - case e, ok = <-nrChan: - } - - if ok { - c.routeEvent(ctx, e) - continue - } - - if !c.reconnect(ctx) { - return - } - } -} - -func (c *Client) routeEvent(ctx context.Context, e any) { - typedNotification := rpcclient.Notification{Value: e} - - switch e.(type) { - case *block.Block: - typedNotification.Type = neorpc.BlockEventID - case *state.ContainedNotificationEvent: - typedNotification.Type = neorpc.NotificationEventID - case *result.NotaryRequestEvent: - typedNotification.Type = neorpc.NotaryRequestEventID - } - +func (c *Client) closeWaiter(ctx context.Context) { select { - case c.notifications <- typedNotification: case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() } -} - -func (c *Client) reconnect(ctx context.Context) bool { - if closeErr := c.client.GetError(); closeErr != nil { - c.logger.Warn(logs.ClientSwitchingToTheNextRPCNode, - zap.String("reason", closeErr.Error()), - ) - } else { - // neo-go client was closed by calling `Close` - // method, that happens only when a client has - // switched to the more prioritized RPC - return true - } - - if !c.switchRPC(ctx) { - c.logger.Error(logs.ClientCouldNotEstablishConnectionToAnyRPCNode) - - // could not connect to all endpoints => - // switch client to inactive mode - c.inactiveMode() - - return false - } - - // TODO(@carpawell): call here some callback retrieved in constructor - // of the client to allow checking chain state since during switch - // process some notification could be lost - - return true + _ = c.UnsubscribeAll() + c.close() } func (c *Client) switchToMostPrioritized(ctx context.Context) { @@ -218,36 +126,28 @@ mainLoop: continue } - if subs, ok := c.restoreSubscriptions(ctx, cli, tryE, true); ok { - c.switchLock.Lock() - - // higher priority node could have been - // connected in the other goroutine - if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { - cli.Close() - c.switchLock.Unlock() - return - } - - c.client.Close() - c.cache.invalidate() - c.client = cli - c.setActor(act) - c.subsInfo = subs - c.endpoints.curr = i + c.switchLock.Lock() + // higher priority node could have been + // connected in the other goroutine + if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { + cli.Close() c.switchLock.Unlock() - - c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, - zap.String("endpoint", tryE)) - return } - c.logger.Warn(logs.ClientCouldNotRestoreSideChainSubscriptionsUsingNode, - zap.String("endpoint", tryE), - zap.Error(err), - ) + c.client.Close() + c.cache.invalidate() + c.client = cli + c.setActor(act) + c.endpoints.curr = i + + c.switchLock.Unlock() + + c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, + zap.String("endpoint", tryE)) + + return } } } @@ -255,6 +155,7 @@ mainLoop: // close closes notification channel and wrapped WS client. func (c *Client) close() { - close(c.notifications) + c.switchLock.RLock() + defer c.switchLock.RUnlock() c.client.Close() } diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index 69eafc659..dbca00d7c 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -1,16 +1,11 @@ package client import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" - "go.uber.org/zap" ) // Close closes connection to the remote side making @@ -24,71 +19,46 @@ func (c *Client) Close() { close(c.closeChan) } -// SubscribeForExecutionNotifications adds subscription for notifications -// generated during contract transaction execution to this instance of client. +// ReceiveExecutionNotifications performs subscription for notifications +// generated during contract execution. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForExecutionNotifications(contract util.Uint160) error { +func (c *Client) ReceiveExecutionNotifications(contract util.Uint160, ch chan<- *state.ContainedNotificationEvent) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, c.notificationRcv) - if err != nil { - return err - } - - c.subscribedEvents[contract] = id - - return nil + return c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, ch) } -// SubscribeForNewBlocks adds subscription for new block events to this -// instance of client. +// ReceiveBlocks performs subscription for new block events. Events are sent +// to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNewBlocks() error { +func (c *Client) ReceiveBlocks(ch chan<- *block.Block) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - if c.subscribedToBlocks { - // no need to subscribe one more time - return nil - } - - _, err := c.client.ReceiveBlocks(nil, c.blockRcv) - if err != nil { - return err - } - - c.subscribedToBlocks = true - - return nil + return c.client.ReceiveBlocks(nil, ch) } -// SubscribeForNotaryRequests adds subscription for notary request payloads +// ReceiveNotaryRequests performsn subscription for notary request payloads // addition or removal events to this instance of client. Passed txSigner is // used as filter: subscription is only for the notary requests that must be -// signed by txSigner. +// signed by txSigner. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { +func (c *Client) ReceiveNotaryRequests(txSigner util.Uint160, ch chan<- *result.NotaryRequestEvent) (string, error) { if c.notary == nil { panic(notaryNotEnabledPanicMsg) } @@ -97,30 +67,17 @@ func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedNotaryEvents[txSigner] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, c.notaryReqRcv) - if err != nil { - return err - } - - c.subscribedNotaryEvents[txSigner] = id - - return nil + return c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, ch) } -// UnsubscribeContract removes subscription for given contract event stream. +// Unsubscribe performs unsubscription for the given subscription ID. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeContract(contract util.Uint160) error { +func (c *Client) Unsubscribe(subID string) error { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -128,55 +85,7 @@ func (c *Client) UnsubscribeContract(contract util.Uint160) error { return ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if !subscribed { - // no need to unsubscribe contract - // without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedEvents[contract]) - if err != nil { - return err - } - - delete(c.subscribedEvents, contract) - - return nil -} - -// UnsubscribeNotaryRequest removes subscription for given notary requests -// signer. -// -// Returns ErrConnectionLost if client has not been able to establish -// connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeNotaryRequest(signer util.Uint160) error { - if c.notary == nil { - panic(notaryNotEnabledPanicMsg) - } - - c.switchLock.Lock() - defer c.switchLock.Unlock() - - if c.inactive { - return ErrConnectionLost - } - - _, subscribed := c.subscribedNotaryEvents[signer] - if !subscribed { - // no need to unsubscribe signer's - // requests without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedNotaryEvents[signer]) - if err != nil { - return err - } - - delete(c.subscribedNotaryEvents, signer) - - return nil + return c.client.Unsubscribe(subID) } // UnsubscribeAll removes all active subscriptions of current client. @@ -191,163 +100,10 @@ func (c *Client) UnsubscribeAll() error { return ErrConnectionLost } - // no need to unsubscribe if there are - // no active subscriptions - if len(c.subscribedEvents) == 0 && len(c.subscribedNotaryEvents) == 0 && - !c.subscribedToBlocks { - return nil - } - err := c.client.UnsubscribeAll() if err != nil { return err } - c.subscribedEvents = make(map[util.Uint160]string) - c.subscribedNotaryEvents = make(map[util.Uint160]string) - c.subscribedToBlocks = false - return nil } - -// subsInfo includes channels for ws notifications; -// cached subscription information. -type subsInfo struct { - blockRcv chan *block.Block - notificationRcv chan *state.ContainedNotificationEvent - notaryReqRcv chan *result.NotaryRequestEvent - - subscribedToBlocks bool - subscribedEvents map[util.Uint160]string - subscribedNotaryEvents map[util.Uint160]string -} - -// restoreSubscriptions restores subscriptions according to cached -// information about them. -// -// If it is NOT a background operation switchLock MUST be held. -// Returns a pair: the second is a restoration status and the first -// one contains subscription information applied to the passed cli -// and receivers for the updated subscriptions. -// Does not change Client instance. -func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClient, endpoint string, background bool) (si subsInfo, ok bool) { - var ( - err error - id string - ) - - stopCh := make(chan struct{}) - defer close(stopCh) - - blockRcv := make(chan *block.Block) - notificationRcv := make(chan *state.ContainedNotificationEvent) - notaryReqRcv := make(chan *result.NotaryRequestEvent) - - c.startListen(ctx, stopCh, blockRcv, notificationRcv, notaryReqRcv, background) - - if background { - c.switchLock.RLock() - defer c.switchLock.RUnlock() - } - - si.subscribedToBlocks = c.subscribedToBlocks - si.subscribedEvents = copySubsMap(c.subscribedEvents) - si.subscribedNotaryEvents = copySubsMap(c.subscribedNotaryEvents) - si.blockRcv = blockRcv - si.notificationRcv = notificationRcv - si.notaryReqRcv = notaryReqRcv - - // new block events restoration - if si.subscribedToBlocks { - _, err = cli.ReceiveBlocks(nil, blockRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - } - - // notification events restoration - for contract := range si.subscribedEvents { - contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, notificationRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedEvents[contract] = id - } - - // notary notification events restoration - if c.notary != nil { - for signer := range si.subscribedNotaryEvents { - signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &signer}, notaryReqRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedNotaryEvents[signer] = id - } - } - - return si, true -} - -func (c *Client) startListen(ctx context.Context, stopCh <-chan struct{}, blockRcv <-chan *block.Block, - notificationRcv <-chan *state.ContainedNotificationEvent, notaryReqRcv <-chan *result.NotaryRequestEvent, background bool) { - // neo-go WS client says to _always_ read notifications - // from its channel. Subscribing to any notification - // while not reading them in another goroutine may - // lead to a dead-lock, thus that async side notification - // listening while restoring subscriptions - - go func() { - var e any - var ok bool - - for { - select { - case <-stopCh: - return - case e, ok = <-blockRcv: - case e, ok = <-notificationRcv: - case e, ok = <-notaryReqRcv: - } - - if !ok { - return - } - - if background { - // background client (test) switch, no need to send - // any notification, just preventing dead-lock - continue - } - - c.routeEvent(ctx, e) - } - }() -} - -func copySubsMap(m map[util.Uint160]string) map[util.Uint160]string { - newM := make(map[util.Uint160]string, len(m)) - for k, v := range m { - newM[k] = v - } - - return newM -} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 2d478c9c3..383d58407 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -35,16 +35,27 @@ type ( Close() } + subChannels struct { + NotifyChan chan *state.ContainedNotificationEvent + BlockChan chan *block.Block + NotaryChan chan *result.NotaryRequestEvent + } + subscriber struct { *sync.RWMutex log *logger.Logger client *client.Client notifyChan chan *state.ContainedNotificationEvent - - blockChan chan *block.Block - + blockChan chan *block.Block notaryChan chan *result.NotaryRequestEvent + + current subChannels + + // cached subscription information + subscribedEvents map[util.Uint160]bool + subscribedNotaryEvents map[util.Uint160]bool + subscribedToNewBlocks bool } // Params is a group of Subscriber constructor parameters. @@ -75,22 +86,28 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { s.Lock() defer s.Unlock() - notifyIDs := make(map[util.Uint160]struct{}, len(contracts)) + notifyIDs := make([]string, 0, len(contracts)) for i := range contracts { + if s.subscribedEvents[contracts[i]] { + continue + } // subscribe to contract notifications - err := s.client.SubscribeForExecutionNotifications(contracts[i]) + id, err := s.client.ReceiveExecutionNotifications(contracts[i], s.current.NotifyChan) if err != nil { // if there is some error, undo all subscriptions and return error - for hash := range notifyIDs { - _ = s.client.UnsubscribeContract(hash) + for _, id := range notifyIDs { + _ = s.client.Unsubscribe(id) } return err } // save notification id - notifyIDs[contracts[i]] = struct{}{} + notifyIDs = append(notifyIDs, id) + } + for i := range contracts { + s.subscribedEvents[contracts[i]] = true } return nil @@ -101,82 +118,34 @@ func (s *subscriber) Close() { } func (s *subscriber) BlockNotifications() error { - if err := s.client.SubscribeForNewBlocks(); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedToNewBlocks { + return nil + } + if _, err := s.client.ReceiveBlocks(s.current.BlockChan); err != nil { return fmt.Errorf("could not subscribe for new block events: %w", err) } + s.subscribedToNewBlocks = true + return nil } func (s *subscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { - if err := s.client.SubscribeForNotaryRequests(mainTXSigner); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedNotaryEvents[mainTXSigner] { + return nil + } + if _, err := s.client.ReceiveNotaryRequests(mainTXSigner, s.current.NotaryChan); err != nil { return fmt.Errorf("could not subscribe for notary request events: %w", err) } + s.subscribedNotaryEvents[mainTXSigner] = true return nil } -func (s *subscriber) routeNotifications(ctx context.Context) { - notificationChan := s.client.NotificationChannel() - - for { - select { - case <-ctx.Done(): - return - case notification, ok := <-notificationChan: - if !ok { - s.log.Warn(logs.SubscriberRemoteNotificationChannelHasBeenClosed) - close(s.notifyChan) - close(s.blockChan) - close(s.notaryChan) - - return - } - - switch notification.Type { - case neorpc.NotificationEventID: - notifyEvent, ok := notification.Value.(*state.ContainedNotificationEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotifyStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.log.Debug(logs.SubscriberNewNotificationEventFromSidechain, - zap.String("name", notifyEvent.Name), - ) - - s.notifyChan <- notifyEvent - case neorpc.BlockEventID: - b, ok := notification.Value.(*block.Block) - if !ok { - s.log.Error(logs.SubscriberCantCastBlockEventValueToBlock, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.blockChan <- b - case neorpc.NotaryRequestEventID: - notaryRequest, ok := notification.Value.(*result.NotaryRequestEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.notaryChan <- notaryRequest - default: - s.log.Debug(logs.SubscriberUnsupportedNotificationFromTheChain, - zap.Uint8("type", uint8(notification.Type)), - ) - } - } - } -} - // New is a constructs Neo:Morph event listener and returns Subscriber interface. func New(ctx context.Context, p *Params) (Subscriber, error) { switch { @@ -200,16 +169,170 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { notifyChan: make(chan *state.ContainedNotificationEvent), blockChan: make(chan *block.Block), notaryChan: make(chan *result.NotaryRequestEvent), - } - // Worker listens all events from neo-go websocket and puts them - // into corresponding channel. It may be notifications, transactions, - // new blocks. For now only notifications. + current: newSubChannels(), + + subscribedEvents: make(map[util.Uint160]bool), + subscribedNotaryEvents: make(map[util.Uint160]bool), + } + // Worker listens all events from temporary NeoGo channel and puts them + // into corresponding permanent channels. go sub.routeNotifications(ctx) return sub, nil } +func (s *subscriber) routeNotifications(ctx context.Context) { + var ( + // TODO: not needed after nspcc-dev/neo-go#2980. + cliCh = s.client.NotificationChannel() + restoreCh = make(chan bool) + restoreInProgress bool + ) + +routeloop: + for { + var connLost bool + s.RLock() + curr := s.current + s.RUnlock() + select { + case <-ctx.Done(): + break routeloop + case ev, ok := <-curr.NotifyChan: + if ok { + s.notifyChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.BlockChan: + if ok { + s.blockChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.NotaryChan: + if ok { + s.notaryChan <- ev + } else { + connLost = true + } + case _, ok := <-cliCh: + connLost = !ok + case ok := <-restoreCh: + restoreInProgress = false + if !ok { + connLost = true + } + } + if connLost { + if !restoreInProgress { + restoreInProgress, cliCh = s.switchEndpoint(ctx, restoreCh) + if !restoreInProgress { + break routeloop + } + curr.drain() + } else { // Avoid getting additional !ok events. + s.Lock() + s.current.NotifyChan = nil + s.current.BlockChan = nil + s.current.NotaryChan = nil + s.Unlock() + } + } + } + close(s.notifyChan) + close(s.blockChan) + close(s.notaryChan) +} + +func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) (bool, <-chan rpcclient.Notification) { + s.log.Info("RPC connection lost, attempting reconnect") + if !s.client.SwitchRPC(ctx) { + s.log.Error("can't switch RPC node") + return false, nil + } + + cliCh := s.client.NotificationChannel() + + s.Lock() + chs := newSubChannels() + go func() { + finishCh <- s.restoreSubscriptions(chs.NotifyChan, chs.BlockChan, chs.NotaryChan) + }() + s.current = chs + s.Unlock() + + return true, cliCh +} + +func newSubChannels() subChannels { + return subChannels{ + NotifyChan: make(chan *state.ContainedNotificationEvent), + BlockChan: make(chan *block.Block), + NotaryChan: make(chan *result.NotaryRequestEvent), + } +} + +func (s *subChannels) drain() { +drainloop: + for { + select { + case _, ok := <-s.NotifyChan: + if !ok { + s.NotifyChan = nil + } + case _, ok := <-s.BlockChan: + if !ok { + s.BlockChan = nil + } + case _, ok := <-s.NotaryChan: + if !ok { + s.NotaryChan = nil + } + default: + break drainloop + } + } +} + +// restoreSubscriptions restores subscriptions according to +// cached information about them. +func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotificationEvent, + blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent) bool { + var err error + + // new block events restoration + if s.subscribedToNewBlocks { + _, err = s.client.ReceiveBlocks(blCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notification events restoration + for contract := range s.subscribedEvents { + contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notary notification events restoration + for signer := range s.subscribedNotaryEvents { + signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + return true +} + // awaitHeight checks if remote client has least expected block height and // returns error if it is not reached that height after timeout duration. // This function is required to avoid connections to unsynced RPC nodes, because -- 2.45.2 From ddcc156eccc3e82dc50738bc5f3aef9bb50896a2 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 18 Apr 2023 18:40:20 +0300 Subject: [PATCH 006/233] [#337] morph: Use Notary Actor for notary requests Signed-off-by: Anna Shaleva Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 2 - pkg/morph/client/notary.go | 394 ++++++++-------------- pkg/morph/event/notary_preparator.go | 25 +- pkg/morph/event/notary_preparator_test.go | 196 +++++++---- 4 files changed, 281 insertions(+), 336 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 832315018..284e065fb 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -153,8 +153,6 @@ func (e *notHaltStateError) Error() string { ) } -var errEmptyInvocationScript = errors.New("got empty invocation script from neo node") - // implementation of error interface for FrostFS-specific errors. type frostfsError struct { err error diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3e21911e1..e478a5118 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -1,6 +1,7 @@ package client import ( + "crypto/elliptic" "encoding/binary" "errors" "fmt" @@ -18,10 +19,12 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/notary" sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "go.uber.org/zap" ) @@ -408,32 +411,32 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return fmt.Errorf("could not fetch current alphabet keys: %w", err) } - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) + cosigners, err := c.notaryCosignersFromTx(mainTx, alphabetList) if err != nil { return err } - // mainTX is expected to be pre-validated: second witness must exist and be empty - mainTx.Scripts[1].VerificationScript = multiaddrAccount.GetVerificationScript() - mainTx.Scripts[1].InvocationScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddrAccount.SignHashable(c.rpcActor.GetNetwork(), mainTx)..., - ) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + // Sign exactly the same transaction we've got from the received Notary request. + err = nAct.Sign(mainTx) + if err != nil { + return fmt.Errorf("faield to sign notary request: %w", err) + } + + mainH, fbH, untilActual, err := nAct.Notarize(mainTx, nil) - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestWithPreparedMainTXInvoked, - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.String("tx_hash", mainH.StringLE()), + zap.Uint32("valid_until_block", untilActual), + zap.String("fallback_hash", fbH.StringLE())) return nil } @@ -449,70 +452,147 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return err } - cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) - if err != nil { - return err - } - - params, err := invocationParams(args...) - if err != nil { - return err - } - - test, err := c.makeTestInvocation(contract, method, params, cosigners) - if err != nil { - return err - } - - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, committee, invokedByAlpha) - if err != nil { - return err - } - until, err := c.getUntilValue(vub) if err != nil { return err } - mainTx, err := c.buildMainTx(invokedByAlpha, nonce, alphabetList, test, cosigners, multiaddrAccount, until) + cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { return err } - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { + if r.State != vmstate.Halt.String() { + return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + } + + t.ValidUntilBlock = until + t.Nonce = nonce + + return nil + }, args...)) + if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestInvoked, zap.String("method", method), - zap.Uint32("valid_until_block", until), - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.Uint32("valid_until_block", untilActual), + zap.String("tx_hash", mainH.StringLE()), + zap.String("fallback_hash", fbH.StringLE())) return nil } -func (c *Client) makeTestInvocation(contract util.Uint160, method string, params []sc.Parameter, cosigners []transaction.Signer) (*result.Invoke, error) { - test, err := c.client.InvokeFunction(contract, method, params, cosigners) +func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabetList keys.PublicKeys) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) if err != nil { return nil, err } - if test.State != HaltState { - return nil, wrapFrostFSError(¬HaltStateError{state: test.State, exception: test.FaultException}) + // Here we need to add a committee signature (second witness) to the pre-validated + // main transaction without creating a new one. However, Notary actor demands the + // proper set of signers for constructor, thus, fill it from the main transaction's signers list. + s := make([]actor.SignerAccount, 2, 3) + s[0] = actor.SignerAccount{ + // Proxy contract that will pay for the execution. + Signer: mainTx.Signers[0], + Account: notary.FakeContractAccount(mainTx.Signers[0].Account), + } + s[1] = actor.SignerAccount{ + // Inner ring multisignature. + Signer: mainTx.Signers[1], + Account: multiaddrAccount, + } + if len(mainTx.Signers) > 3 { + // Invoker signature (simple signature account of storage node is expected). + var acc *wallet.Account + script := mainTx.Scripts[2].VerificationScript + if len(script) == 0 { + acc = notary.FakeContractAccount(mainTx.Signers[2].Account) + } else { + pubBytes, ok := vm.ParseSignatureContract(script) + if ok { + pub, err := keys.NewPublicKeyFromBytes(pubBytes, elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key: %w", err) + } + acc = notary.FakeSimpleAccount(pub) + } else { + m, pubsBytes, ok := vm.ParseMultiSigContract(script) + if !ok { + return nil, errors.New("failed to parse verification script of signer #2: unknown witness type") + } + pubs := make(keys.PublicKeys, len(pubsBytes)) + for i := range pubs { + pubs[i], err = keys.NewPublicKeyFromBytes(pubsBytes[i], elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key #%d: %w", i, err) + } + } + acc, err = notary.FakeMultisigAccount(m, pubs) + if err != nil { + return nil, fmt.Errorf("failed to create fake account for signer #2: %w", err) + } + } + } + s = append(s, actor.SignerAccount{ + Signer: mainTx.Signers[2], + Account: acc, + }) } - if len(test.Script) == 0 { - return nil, wrapFrostFSError(errEmptyInvocationScript) + return s, nil +} + +func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(ir, committee, invokedByAlpha) + if err != nil { + return nil, err } - return test, nil + s := make([]actor.SignerAccount, 2, 3) + // Proxy contract that will pay for the execution. + s[0] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: c.notary.proxy, + Scopes: transaction.None, + }, + Account: notary.FakeContractAccount(c.notary.proxy), + } + // Inner ring multisignature. + s[1] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: multiaddrAccount.ScriptHash(), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: multiaddrAccount, + } + + if !invokedByAlpha { + // Invoker signature. + s = append(s, actor.SignerAccount{ + Signer: transaction.Signer{ + Account: hash.Hash160(c.acc.GetVerificationScript()), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: c.acc, + }) + } + + // The last one is Notary contract that will be added to the signers list + // by Notary actor automatically. + return s, nil } func (c *Client) getUntilValue(vub *uint32) (uint32, error) { @@ -522,195 +602,6 @@ func (c *Client) getUntilValue(vub *uint32) (uint32, error) { return c.notaryTxValidationLimit() } -func (c *Client) buildMainTx(invokedByAlpha bool, nonce uint32, alphabetList keys.PublicKeys, test *result.Invoke, - cosigners []transaction.Signer, multiaddrAccount *wallet.Account, until uint32) (*transaction.Transaction, error) { - // after test invocation we build main multisig transaction - - u8n := uint8(len(alphabetList)) - - if !invokedByAlpha { - u8n++ - } - - // prepare main tx - mainTx := &transaction.Transaction{ - Nonce: nonce, - SystemFee: test.GasConsumed, - ValidUntilBlock: until, - Script: test.Script, - Attributes: []transaction.Attribute{ - { - Type: transaction.NotaryAssistedT, - Value: &transaction.NotaryAssisted{NKeys: u8n}, - }, - }, - Signers: cosigners, - } - - // calculate notary fee - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - notaryFee, err := c.client.CalculateNotaryFee(u8n) - if err != nil { - return nil, err - } - - // add network fee for cosigners - //nolint:staticcheck // waits for neo-go v0.99.3 with notary actors - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - err = c.client.AddNetworkFee( - mainTx, - notaryFee, - c.notaryAccounts(invokedByAlpha, multiaddrAccount)..., - ) - if err != nil { - return nil, err - } - - // define witnesses - mainTx.Scripts = c.notaryWitnesses(invokedByAlpha, multiaddrAccount, mainTx) - - return mainTx, nil -} - -func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]transaction.Signer, error) { - s := make([]transaction.Signer, 0, 4) - - // first we have proxy contract signature, as it will pay for the execution - s = append(s, transaction.Signer{ - Account: c.notary.proxy, - Scopes: transaction.None, - }) - - // then we have inner ring multiaddress signature - m := sigCount(ir, committee) - - multisigScript, err := sc.CreateMultiSigRedeemScript(m, ir) - if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("can't create ir multisig redeem script: %w", err)) - } - - s = append(s, transaction.Signer{ - Account: hash.Hash160(multisigScript), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - - if !invokedByAlpha { - // then we have invoker signature - s = append(s, transaction.Signer{ - Account: hash.Hash160(c.acc.GetVerificationScript()), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - } - - // last one is a placeholder for notary contract signature - s = append(s, transaction.Signer{ - Account: c.notary.notary, - Scopes: transaction.None, - }) - - return s, nil -} - -func (c *Client) notaryAccounts(invokedByAlpha bool, multiaddr *wallet.Account) []*wallet.Account { - if multiaddr == nil { - return nil - } - - a := make([]*wallet.Account, 0, 4) - - // first we have proxy account, as it will pay for the execution - a = append(a, notary.FakeContractAccount(c.notary.proxy)) - - // then we have inner ring multiaddress account - a = append(a, multiaddr) - - if !invokedByAlpha { - // then we have invoker account - a = append(a, c.acc) - } - - // last one is a placeholder for notary contract account - a = append(a, &wallet.Account{ - Contract: &wallet.Contract{}, - }) - - return a -} - -func (c *Client) notaryWitnesses(invokedByAlpha bool, multiaddr *wallet.Account, tx *transaction.Transaction) []transaction.Witness { - if multiaddr == nil || tx == nil { - return nil - } - - w := make([]transaction.Witness, 0, 4) - - // first we have empty proxy witness, because notary will execute `Verify` - // method on the proxy contract to check witness - w = append(w, transaction.Witness{ - InvocationScript: []byte{}, - VerificationScript: []byte{}, - }) - - // then we have inner ring multiaddress witness - - // invocation script should be of the form: - // { PUSHDATA1, 64, signatureBytes... } - // to pass Notary module verification - var invokeScript []byte - - magicNumber := c.rpcActor.GetNetwork() - - if invokedByAlpha { - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddr.SignHashable(magicNumber, tx)..., - ) - } else { - // we can't provide alphabet node signature - // because Storage Node doesn't own alphabet's - // private key. Thus, add dummy witness with - // empty bytes instead of signature - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ) - } - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: multiaddr.GetVerificationScript(), - }) - - if !invokedByAlpha { - // then we have invoker witness - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - c.acc.SignHashable(magicNumber, tx)..., - ) - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: c.acc.GetVerificationScript(), - }) - } - - // last one is a placeholder for notary contract witness - w = append(w, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ), - VerificationScript: []byte{}, - }) - - return w -} - func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedByAlpha bool) (*wallet.Account, error) { m := sigCount(ir, committee) @@ -767,21 +658,6 @@ func (c *Client) depositExpirationOf() (int64, error) { return currentTillBig.Int64(), nil } -func invocationParams(args ...any) ([]sc.Parameter, error) { - params := make([]sc.Parameter, 0, len(args)) - - for i := range args { - param, err := toStackParameter(args[i]) - if err != nil { - return nil, err - } - - params = append(params, param) - } - - return params, nil -} - // sigCount returns the number of required signature. // For FrostFS Alphabet M is a 2/3+1 of it (like in dBFT). // If committee is true, returns M as N/2+1. diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index f7b10d906..a8b7376fa 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -185,15 +185,15 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } invokerWitness := ln == 4 - multiInvScript := nr.MainTransaction.Scripts[1].InvocationScript - - // alphabet node should handle only notary requests - // that have been sent unsigned (by storage nodes) => - // such main TXs should have either a dummy or an - // empty script as an invocation script + // alphabet node should handle only notary requests that do not yet have inner + // ring multisignature filled => such main TXs either have empty invocation script + // of the inner ring witness (in case if Notary Actor is used to create request) + // or have it filled with dummy bytes (if request was created manually with the old + // neo-go API) // // this check prevents notary flow recursion - if len(multiInvScript) > 0 && !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { + if !(len(nr.MainTransaction.Scripts[1].InvocationScript) == 0 || + bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript)) { // compatibility with old version return ErrTXAlreadyHandled } @@ -220,12 +220,7 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } // validate main TX expiration - err = p.validateExpiration(nr.FallbackTransaction) - if err != nil { - return err - } - - return nil + return p.validateExpiration(nr.FallbackTransaction) } func (p Preparator) validateParameterOpcodes(ops []Op) error { @@ -363,7 +358,9 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu // the last one must be a placeholder for notary contract witness last := len(w) - 1 - if !bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript) || len(w[last].VerificationScript) != 0 { + if !(len(w[last].InvocationScript) == 0 || // https://github.com/nspcc-dev/neo-go/pull/2981 + bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version + len(w[last].VerificationScript) != 0 { return errIncorrectNotaryPlaceholder } diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index d0463348d..b2e46890b 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -1,6 +1,7 @@ package event import ( + "fmt" "testing" "github.com/nspcc-dev/neo-go/pkg/vm" @@ -24,8 +25,9 @@ var ( alphaKeys keys.PublicKeys wrongAlphaKeys keys.PublicKeys - dummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) - wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) + dummyAlphabetInvocationScript = []byte{} // expected to be empty if generated by Notary Actor, as requester can't fill it in + dummyAlphabetInvocationScriptOld = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) // expected to be dummy if generated manually + wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) scriptHash util.Uint160 ) @@ -61,35 +63,37 @@ func TestPrepare_IncorrectScript(t *testing.T) { }, ) - t.Run("not contract call", func(t *testing.T) { - bw := io.NewBufBinWriter() + for _, dummyMultisig := range []bool{true, false} { // try both empty and dummy multisig/Notary invocation witness script + t.Run(fmt.Sprintf("not contract call, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, 4) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall + emit.Int(bw.BinWriter, 4) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errNotContractCall.Error()) - }) + require.EqualError(t, err, errNotContractCall.Error()) + }) - t.Run("incorrect ", func(t *testing.T) { - bw := io.NewBufBinWriter() + t.Run(fmt.Sprintf("incorrect, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, -1) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) + emit.Int(bw.BinWriter, -1) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errIncorrectCallFlag.Error()) - }) + require.EqualError(t, err, errIncorrectCallFlag.Error()) + }) + } } func TestPrepare_IncorrectNR(t *testing.T) { @@ -209,7 +213,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { InvocationScript: make([]byte, 1), }, { - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectProxyWitnesses, + }, + { + name: "incorrect main TX proxy witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + { + InvocationScript: make([]byte, 1), + }, + { + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -224,7 +244,22 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: wrongAlphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectAlphabet, + }, + { + name: "incorrect main TX Alphabet witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: wrongAlphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -239,7 +274,24 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + { + InvocationScript: wrongDummyInvocationScript, + }, + }, + }, + expErr: errIncorrectNotaryPlaceholder, + }, + { + name: "incorrect main TX Notary witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, { InvocationScript: wrongDummyInvocationScript, @@ -289,7 +341,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + {}, + }, + }, + expErr: errIncorrectInvokerWitnesses, + }, + { + name: "incorrect invoker TX Alphabet witness compat", + addW: true, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, {}, @@ -327,7 +395,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - correctNR := correctNR(nil, test.addW) + correctNR := correctNR(nil, false, test.addW) incorrectNR = setIncorrectFields(*correctNR, test.mTX, test.fbTX) _, err = preparator.Prepare(&incorrectNR) @@ -372,40 +440,42 @@ func TestPrepare_CorrectNR(t *testing.T) { for _, test := range tests { for i := 0; i < 1; i++ { // run tests against 3 and 4 witness NR - additionalWitness := i == 0 - nr := correctNR(script(test.hash, test.method, test.args...), additionalWitness) + for _, dummyMultisig := range []bool{true, false} { // run tests against empty and dummy multisig/Notary witness + additionalWitness := i == 0 + nr := correctNR(script(test.hash, test.method, test.args...), dummyMultisig, additionalWitness) - event, err := preparator.Prepare(nr) + event, err := preparator.Prepare(nr) - require.NoError(t, err) - require.Equal(t, test.method, event.Type().String()) - require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) - - // check args parsing - bw := io.NewBufBinWriter() - emit.Array(bw.BinWriter, test.args...) - - ctx := vm.NewContext(bw.Bytes()) - - opCode, param, err := ctx.Next() - require.NoError(t, err) - - for _, opGot := range event.Params() { - require.Equal(t, opCode, opGot.code) - require.Equal(t, param, opGot.param) - - opCode, param, err = ctx.Next() require.NoError(t, err) + require.Equal(t, test.method, event.Type().String()) + require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) + + // check args parsing + bw := io.NewBufBinWriter() + emit.Array(bw.BinWriter, test.args...) + + ctx := vm.NewContext(bw.Bytes()) + + opCode, param, err := ctx.Next() + require.NoError(t, err) + + for _, opGot := range event.Params() { + require.Equal(t, opCode, opGot.code) + require.Equal(t, param, opGot.param) + + opCode, param, err = ctx.Next() + require.NoError(t, err) + } + + _, _, err = ctx.Next() // PACK opcode + require.NoError(t, err) + _, _, err = ctx.Next() // packing len opcode + require.NoError(t, err) + + opCode, _, err = ctx.Next() + require.NoError(t, err) + require.Equal(t, opcode.RET, opCode) } - - _, _, err = ctx.Next() // PACK opcode - require.NoError(t, err) - _, _, err = ctx.Next() // packing len opcode - require.NoError(t, err) - - opCode, _, err = ctx.Next() - require.NoError(t, err) - require.Equal(t, opcode.RET, opCode) } } } @@ -428,7 +498,7 @@ func script(hash util.Uint160, method string, args ...any) []byte { return bw.Bytes() } -func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest { +func correctNR(script []byte, dummyMultisig, additionalWitness bool) *payload.P2PNotaryRequest { alphaVerificationScript, _ := smartcontract.CreateMultiSigRedeemScript(len(alphaKeys)*2/3+1, alphaKeys) signers := []transaction.Signer{ @@ -443,20 +513,24 @@ func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest signers[2] = transaction.Signer{Account: hash.Hash160(alphaVerificationScript)} } + multisigInv := dummyAlphabetInvocationScript + if dummyMultisig { + multisigInv = dummyAlphabetInvocationScriptOld + } scripts := []transaction.Witness{ {}, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, }, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, }, } if additionalWitness { // insert on element with index 2 scripts = append(scripts[:2+1], scripts[2:]...) scripts[2] = transaction.Witness{ - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, } } -- 2.45.2 From bcdb0f330dd0d3b51ddbe0d02cccfe1af562d6c7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 20 Apr 2023 18:53:48 +0300 Subject: [PATCH 007/233] [#337] morph: Completely remove fallbackTime from client cfg It's unused and not needed, default fallback lifetime is set by Notary actor. Signed-off-by: Anna Shaleva Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index e478a5118..1ed1ca912 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -31,9 +31,8 @@ import ( type ( notaryInfo struct { - txValidTime uint32 // minimum amount of blocks when mainTx will be valid - roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes - fallbackTime uint32 // mainTx's ValidUntilBlock - fallbackTime + 1 is when fallbackTx is sent + txValidTime uint32 // minimum amount of blocks when mainTx will be valid + roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes alphabetSource AlphabetKeys // source of alphabet node keys to prepare witness @@ -44,7 +43,7 @@ type ( notaryCfg struct { proxy util.Uint160 - txValidTime, roundTime, fallbackTime uint32 + txValidTime, roundTime uint32 alphabetSource AlphabetKeys } @@ -54,9 +53,8 @@ type ( ) const ( - defaultNotaryValidTime = 50 - defaultNotaryRoundTime = 100 - defaultNotaryFallbackTime = 40 + defaultNotaryValidTime = 50 + defaultNotaryRoundTime = 100 notaryBalanceOfMethod = "balanceOf" notaryExpirationOfMethod = "expirationOf" @@ -72,7 +70,6 @@ func defaultNotaryConfig(c *Client) *notaryCfg { return ¬aryCfg{ txValidTime: defaultNotaryValidTime, roundTime: defaultNotaryRoundTime, - fallbackTime: defaultNotaryFallbackTime, alphabetSource: c.Committee, } } @@ -107,7 +104,6 @@ func (c *Client) EnableNotarySupport(opts ...NotaryOption) error { proxy: cfg.proxy, txValidTime: cfg.txValidTime, roundTime: cfg.roundTime, - fallbackTime: cfg.fallbackTime, alphabetSource: cfg.alphabetSource, notary: notary.Hash, } @@ -685,15 +681,6 @@ func WithRoundTime(t uint32) NotaryOption { } } -// WithFallbackTime returns a notary support option for client -// that specifies amount of blocks before fallbackTx will be sent. -// Should be less than TxValidTime. -func WithFallbackTime(t uint32) NotaryOption { - return func(c *notaryCfg) { - c.fallbackTime = t - } -} - // WithAlphabetSource returns a notary support option for client // that specifies function to return list of alphabet node keys. // By default notary subsystem uses committee as a source. This is -- 2.45.2 From b480df4985a1be3cf7914638783a369a1f8f6f00 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 18:14:00 +0300 Subject: [PATCH 008/233] [#337] container: Remove unused events Done in TrueCloudLab/frostfs-contract#16. Signed-off-by: Evgenii Stratonikov --- .../processors/container/processor.go | 65 +-------------- pkg/morph/event/container/delete.go | 39 --------- pkg/morph/event/container/delete_test.go | 60 -------------- pkg/morph/event/container/eacl.go | 50 ------------ pkg/morph/event/container/eacl_test.go | 80 ------------------- pkg/morph/event/container/put.go | 43 ---------- pkg/morph/event/container/put_test.go | 74 ----------------- 7 files changed, 2 insertions(+), 409 deletions(-) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index d5af5e394..2141b0764 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -79,13 +79,6 @@ type NetworkState interface { HomomorphicHashDisabled() (bool, error) } -const ( - putNotification = "containerPut" - deleteNotification = "containerDelete" - - setEACLNotification = "setEACL" -) - // New creates a container contract processor instance. func New(p *Params) (*Processor, error) { switch { @@ -121,66 +114,12 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationParsers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - parsers = make([]event.NotificationParserInfo, 0, 3) - - p event.NotificationParserInfo - ) - - p.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - p.SetType(event.TypeFromString(putNotification)) - p.SetParser(containerEvent.ParsePut) - parsers = append(parsers, p) - - // container delete - p.SetType(event.TypeFromString(deleteNotification)) - p.SetParser(containerEvent.ParseDelete) - parsers = append(parsers, p) - - // set eACL - p.SetType(event.TypeFromString(setEACLNotification)) - p.SetParser(containerEvent.ParseSetEACL) - parsers = append(parsers, p) - - return parsers + return nil } // ListenerNotificationHandlers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - handlers = make([]event.NotificationHandlerInfo, 0, 3) - - h event.NotificationHandlerInfo - ) - - h.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - h.SetType(event.TypeFromString(putNotification)) - h.SetHandler(cp.handlePut) - handlers = append(handlers, h) - - // container delete - h.SetType(event.TypeFromString(deleteNotification)) - h.SetHandler(cp.handleDelete) - handlers = append(handlers, h) - - // set eACL - h.SetType(event.TypeFromString(setEACLNotification)) - h.SetHandler(cp.handleSetEACL) - handlers = append(handlers, h) - - return handlers + return nil } // ListenerNotaryParsers for the 'event.Listener' notary event producer. diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 7286ddcfc..4926af27d 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -44,45 +44,6 @@ func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { const expectedItemNumDelete = 3 -// ParseDelete from notification into container event structure. -// -// Expects 3 stack items. -func ParseDelete(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Delete - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumDelete { - return nil, event.WrongNumberOfParameters(expectedItemNumDelete, ln) - } - - // parse container - ev.ContainerIDValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} - // DeleteSuccess structures notification event of successful container removal // thrown by Container contract. type DeleteSuccess struct { diff --git a/pkg/morph/event/container/delete_test.go b/pkg/morph/event/container/delete_test.go index 782f4aade..627c5fcf5 100644 --- a/pkg/morph/event/container/delete_test.go +++ b/pkg/morph/event/container/delete_test.go @@ -10,66 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseDelete(t *testing.T) { - var ( - containerID = []byte("containreID") - signature = []byte("signature") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseDelete(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(token), - })) - - require.NoError(t, err) - - require.Equal(t, Delete{ - ContainerIDValue: containerID, - SignatureValue: signature, - TokenValue: token, - }, ev) - }) -} - func TestParseDeleteSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ diff --git a/pkg/morph/event/container/eacl.go b/pkg/morph/event/container/eacl.go index 41058ea43..4168d8842 100644 --- a/pkg/morph/event/container/eacl.go +++ b/pkg/morph/event/container/eacl.go @@ -1,11 +1,6 @@ package container import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -54,48 +49,3 @@ func (x SetEACL) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumEACL = 4 - -// ParseSetEACL parses SetEACL notification event from list of stack items. -// -// Expects 4 stack items. -func ParseSetEACL(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev SetEACL - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumEACL { - return nil, event.WrongNumberOfParameters(expectedItemNumEACL, ln) - } - - // parse table - ev.TableValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not parse binary table: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not parse table signature: %w", err) - } - - // parse public key - ev.PublicKeyValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not parse binary public key: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/container/eacl_test.go b/pkg/morph/event/container/eacl_test.go index 2f0598597..159f6cd9f 100644 --- a/pkg/morph/event/container/eacl_test.go +++ b/pkg/morph/event/container/eacl_test.go @@ -1,90 +1,10 @@ package container import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseEACL(t *testing.T) { - var ( - binaryTable = []byte("table") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - items := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseSetEACL(createNotifyEventFromItems(items)) - require.EqualError(t, err, event.WrongNumberOfParameters(4, len(items)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - e := ev.(SetEACL) - - require.Equal(t, binaryTable, e.Table()) - require.Equal(t, signature, e.Signature()) - require.Equal(t, publicKey, e.PublicKey()) - require.Equal(t, token, e.SessionToken()) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/container/put.go b/pkg/morph/event/container/put.go index d163c6836..335034bf3 100644 --- a/pkg/morph/event/container/put.go +++ b/pkg/morph/event/container/put.go @@ -65,49 +65,6 @@ func (x PutNamed) Zone() string { return x.zone } -// ParsePut from notification into container event structure. -func ParsePut(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Put - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumPut { - return nil, event.WrongNumberOfParameters(expectedItemNumPut, ln) - } - - // parse container - ev.rawContainer, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.signature, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse public key - ev.publicKey, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - // parse session token - ev.token, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get sesison token: %w", err) - } - - return ev, nil -} - // PutSuccess structures notification event of successful container creation // thrown by Container contract. type PutSuccess struct { diff --git a/pkg/morph/event/container/put_test.go b/pkg/morph/event/container/put_test.go index 2ccea296f..3622f9943 100644 --- a/pkg/morph/event/container/put_test.go +++ b/pkg/morph/event/container/put_test.go @@ -10,80 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParsePut(t *testing.T) { - var ( - containerData = []byte("containerData") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParsePut(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(expectedItemNumPut, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - require.Equal(t, Put{ - rawContainer: containerData, - signature: signature, - publicKey: publicKey, - token: token, - }, ev) - }) -} - func TestParsePutSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ -- 2.45.2 From 47b0ec33c354413116e48a2111e7bb71ca699b31 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 18:17:06 +0300 Subject: [PATCH 009/233] [#337] netmap: Remove unused events Done in TrueCloudLab/frostfs-contract#16. Signed-off-by: Evgenii Stratonikov --- pkg/innerring/processors/netmap/processor.go | 32 +---------- pkg/morph/event/netmap/add_peer.go | 28 ---------- pkg/morph/event/netmap/add_peer_test.go | 37 ------------ pkg/morph/event/netmap/update_peer.go | 44 --------------- pkg/morph/event/netmap/update_peer_test.go | 59 -------------------- 5 files changed, 1 insertion(+), 199 deletions(-) delete mode 100644 pkg/morph/event/netmap/update_peer_test.go diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 522fa8b86..c466cfb1b 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -119,9 +119,7 @@ type ( ) const ( - newEpochNotification = "NewEpoch" - addPeerNotification = "AddPeer" - updatePeerStateNotification = "UpdateState" + newEpochNotification = "NewEpoch" ) // New creates network map contract processor instance. @@ -189,20 +187,6 @@ func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInf p.SetParser(netmapEvent.ParseNewEpoch) parsers = append(parsers, p) - if !np.notaryDisabled { - return parsers - } - - // new peer event - p.SetType(addPeerNotification) - p.SetParser(netmapEvent.ParseAddPeer) - parsers = append(parsers, p) - - // update peer event - p.SetType(updatePeerStateNotification) - p.SetParser(netmapEvent.ParseUpdatePeer) - parsers = append(parsers, p) - return parsers } @@ -219,20 +203,6 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI i.SetHandler(np.handleNewEpoch) handlers = append(handlers, i) - if !np.notaryDisabled { - return handlers - } - - // new peer handler - i.SetType(addPeerNotification) - i.SetHandler(np.handleAddPeer) - handlers = append(handlers, i) - - // update peer handler - i.SetType(updatePeerStateNotification) - i.SetHandler(np.handleUpdateState) - handlers = append(handlers, i) - return handlers } diff --git a/pkg/morph/event/netmap/add_peer.go b/pkg/morph/event/netmap/add_peer.go index 6f839bada..80c5559fc 100644 --- a/pkg/morph/event/netmap/add_peer.go +++ b/pkg/morph/event/netmap/add_peer.go @@ -1,11 +1,6 @@ package netmap import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -31,26 +26,3 @@ func (s AddPeer) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumAddPeer = 1 - -func ParseAddPeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev AddPeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumAddPeer { - return nil, event.WrongNumberOfParameters(expectedItemNumAddPeer, ln) - } - - ev.NodeBytes, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get raw nodeinfo: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/add_peer_test.go b/pkg/morph/event/netmap/add_peer_test.go index 0574c4048..4118bb8c8 100644 --- a/pkg/morph/event/netmap/add_peer_test.go +++ b/pkg/morph/event/netmap/add_peer_test.go @@ -1,47 +1,10 @@ package netmap import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseAddPeer(t *testing.T) { - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseAddPeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - info := []byte{1, 2, 3} - - ev, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(info), - })) - - require.NoError(t, err) - require.Equal(t, AddPeer{ - NodeBytes: info, - }, ev) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/netmap/update_peer.go b/pkg/morph/event/netmap/update_peer.go index f02ca408d..e29671131 100644 --- a/pkg/morph/event/netmap/update_peer.go +++ b/pkg/morph/event/netmap/update_peer.go @@ -1,13 +1,9 @@ package netmap import ( - "crypto/elliptic" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -60,43 +56,3 @@ func (s *UpdatePeer) decodeState(state int64) error { } const expectedItemNumUpdatePeer = 2 - -func ParseUpdatePeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev UpdatePeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumUpdatePeer { - return nil, event.WrongNumberOfParameters(expectedItemNumUpdatePeer, ln) - } - - // parse public key - key, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - ev.PubKey, err = keys.NewPublicKeyFromBytes(key, elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("could not parse public key: %w", err) - } - - // parse node status - st, err := client.IntFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get node status: %w", err) - } - - err = ev.decodeState(st) - if err != nil { - return nil, err - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/update_peer_test.go b/pkg/morph/event/netmap/update_peer_test.go deleted file mode 100644 index b79dd6385..000000000 --- a/pkg/morph/event/netmap/update_peer_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package netmap - -import ( - "math/big" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseUpdatePeer(t *testing.T) { - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - - publicKey := priv.PublicKey() - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseUpdatePeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong second parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(publicKey.Bytes()), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - const state = netmap.NodeStateMaintenance - ev, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(big.NewInt(int64(state))), - stackitem.NewByteArray(publicKey.Bytes()), - })) - require.NoError(t, err) - - require.Equal(t, UpdatePeer{ - PubKey: publicKey, - State: state, - }, ev) - }) -} -- 2.45.2 From 969bfb603f7877eacb2d7c8140ebe001f7785264 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 13:53:33 +0300 Subject: [PATCH 010/233] [#321] shard/test: Parallelize TestShard_List ``` go test -count=1 -run TestShard_List -race . Before: 2.492s After: 0.109s ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/list_test.go | 57 ++++++++++++--------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 8fac41a0f..e0759f668 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -2,6 +2,7 @@ package shard_test import ( "context" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -9,22 +10,21 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestShard_List(t *testing.T) { - sh := newShard(t, false) - shWC := newShard(t, true) - - defer func() { - releaseShard(sh, t) - releaseShard(shWC, t) - }() - t.Run("without write cache", func(t *testing.T) { + t.Parallel() + sh := newShard(t, false) + defer releaseShard(sh, t) testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() + shWC := newShard(t, true) + defer releaseShard(shWC, t) testShardList(t, shWC) }) } @@ -33,30 +33,41 @@ func testShardList(t *testing.T, sh *shard.Shard) { const C = 10 const N = 5 + var mtx sync.Mutex objs := make(map[string]int) - var putPrm shard.PutPrm + var errG errgroup.Group + errG.SetLimit(C * N) for i := 0; i < C; i++ { - cnr := cidtest.ID() + errG.Go(func() error { + cnr := cidtest.ID() - for j := 0; j < N; j++ { - obj := testutil.GenerateObjectWithCID(cnr) - testutil.AddPayload(obj, 1<<2) + for j := 0; j < N; j++ { + errG.Go(func() error { + obj := testutil.GenerateObjectWithCID(cnr) + testutil.AddPayload(obj, 1<<2) - // add parent as virtual object, it must be ignored in List() - parent := testutil.GenerateObjectWithCID(cnr) - idParent, _ := parent.ID() - obj.SetParentID(idParent) - obj.SetParent(parent) + // add parent as virtual object, it must be ignored in List() + parent := testutil.GenerateObjectWithCID(cnr) + idParent, _ := parent.ID() + obj.SetParentID(idParent) + obj.SetParent(parent) - objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Lock() + objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Unlock() - putPrm.SetObject(obj) + var putPrm shard.PutPrm + putPrm.SetObject(obj) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) - } + _, err := sh.Put(context.Background(), putPrm) + return err + }) + } + return nil + }) } + require.NoError(t, errG.Wait()) res, err := sh.List() require.NoError(t, err) -- 2.45.2 From d35e4c389f560552f3868573e082be4d99dd3ab5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 14:07:11 +0300 Subject: [PATCH 011/233] [#321] shard/test: Parallelize TestWriteCacheObjectLoss Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/shutdown_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 714811b7e..947bd73fd 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -12,6 +12,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestWriteCacheObjectLoss(t *testing.T) { @@ -39,13 +40,17 @@ func TestWriteCacheObjectLoss(t *testing.T) { sh := newCustomShard(t, dir, true, wcOpts, nil) - var putPrm shard.PutPrm - + var errG errgroup.Group for i := range objects { - putPrm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) + obj := objects[i] + errG.Go(func() error { + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + return err + }) } + require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) sh = newCustomShard(t, dir, true, wcOpts, nil) -- 2.45.2 From 4578d0061980ba8843ef807c0bc907e803a73fe7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 14:08:10 +0300 Subject: [PATCH 012/233] [#321] shard/test: Execute tests in parallel Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/control_test.go | 6 ++++++ pkg/local_object_storage/shard/delete_test.go | 4 ++++ pkg/local_object_storage/shard/gc_test.go | 2 ++ pkg/local_object_storage/shard/get_test.go | 4 ++++ pkg/local_object_storage/shard/head_test.go | 4 ++++ pkg/local_object_storage/shard/inhume_test.go | 4 ++++ pkg/local_object_storage/shard/list_test.go | 2 ++ pkg/local_object_storage/shard/lock_test.go | 2 ++ pkg/local_object_storage/shard/metrics_test.go | 2 ++ pkg/local_object_storage/shard/range_test.go | 3 +++ pkg/local_object_storage/shard/reload_test.go | 2 ++ pkg/local_object_storage/shard/shutdown_test.go | 2 ++ 12 files changed, 37 insertions(+) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 170052d63..3f0a19cd4 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -43,6 +43,8 @@ type objAddr struct { } func TestShardOpen(t *testing.T) { + t.Parallel() + dir := t.TempDir() metaPath := filepath.Join(dir, "meta") @@ -111,6 +113,8 @@ func TestShardOpen(t *testing.T) { } func TestRefillMetabaseCorrupted(t *testing.T) { + t.Parallel() + dir := t.TempDir() fsTree := fstree.New( @@ -164,6 +168,8 @@ func TestRefillMetabaseCorrupted(t *testing.T) { } func TestRefillMetabase(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 9646e9aa0..441e1c455 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Delete(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, true) }) } diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index b0126fcd7..acc039cd2 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -25,6 +25,8 @@ import ( ) func Test_GCDropsLockedExpiredObject(t *testing.T) { + t.Parallel() + var sh *shard.Shard epoch := &epochState{ diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index ea28c8e32..2db86c48a 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -17,11 +17,15 @@ import ( ) func TestShard_Get(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, true) }) } diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 11e7a8b04..7e336ea06 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -15,11 +15,15 @@ import ( ) func TestShard_Head(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, true) }) } diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 0b4e51701..4151d6218 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Inhume(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, true) }) } diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index e0759f668..bbce28430 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -14,6 +14,8 @@ import ( ) func TestShard_List(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { t.Parallel() sh := newShard(t, false) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index c577ae184..75010179a 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -23,6 +23,8 @@ import ( ) func TestShard_Lock(t *testing.T) { + t.Parallel() + var sh *shard.Shard rootPath := t.TempDir() diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 1578c662b..16f6989c4 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -72,6 +72,8 @@ const physical = "phy" const logical = "logic" func TestCounters(t *testing.T) { + t.Parallel() + dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index c95dbae98..4574ce415 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -22,11 +22,14 @@ import ( ) func TestShard_GetRange(t *testing.T) { + t.Parallel() t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, true) }) } diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 9ad05f525..0b964ba2e 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -24,6 +24,8 @@ import ( ) func TestShardReload(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 947bd73fd..76b20d659 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -16,6 +16,8 @@ import ( ) func TestWriteCacheObjectLoss(t *testing.T) { + t.Parallel() + const ( smallSize = 1024 objCount = 100 -- 2.45.2 From 945454f60c95ce424cb9eb77fccd2f75ea1d98d6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:17:29 +0300 Subject: [PATCH 013/233] [#321] engine/test: Execute tests in parallel Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/delete_test.go | 2 ++ pkg/local_object_storage/engine/evacuate_test.go | 8 ++++++++ pkg/local_object_storage/engine/list_test.go | 6 +++++- pkg/local_object_storage/engine/lock_test.go | 6 ++++++ pkg/local_object_storage/engine/remove_copies_test.go | 4 ++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 53c62981c..bbc27615a 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -18,6 +18,8 @@ import ( ) func TestDeleteBigObject(t *testing.T) { + t.Parallel() + defer os.RemoveAll(t.Name()) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bea6d4ff5..bc5b05ef0 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -77,6 +77,8 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng } func TestEvacuateShard(t *testing.T) { + t.Parallel() + const objPerShard = 3 e, ids, objects := newEngineEvacuate(t, 3, objPerShard) @@ -132,6 +134,8 @@ func TestEvacuateShard(t *testing.T) { } func TestEvacuateNetwork(t *testing.T) { + t.Parallel() + var errReplication = errors.New("handler error") acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { @@ -154,6 +158,7 @@ func TestEvacuateNetwork(t *testing.T) { } t.Run("single shard", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 1, 3) evacuateShardID := ids[0].String() @@ -173,6 +178,7 @@ func TestEvacuateNetwork(t *testing.T) { require.Equal(t, 2, res.Count()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) @@ -195,6 +201,7 @@ func TestEvacuateNetwork(t *testing.T) { }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 4, 5) evacuateIDs := ids[0:3] @@ -229,6 +236,7 @@ func TestEvacuateNetwork(t *testing.T) { } func TestEvacuateCancellation(t *testing.T) { + t.Parallel() e, ids, _ := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 44062be68..5b927cf11 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -29,6 +29,8 @@ func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithTy } func TestListWithCursor(t *testing.T) { + t.Parallel() + tests := []struct { name string shardNum int @@ -60,8 +62,10 @@ func TestListWithCursor(t *testing.T) { batchSize: 100, }, } - for _, tt := range tests { + for i := range tests { + tt := tests[i] t.Run(tt.name, func(t *testing.T) { + t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zap.L()}), diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 4c89b9226..7796913ea 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -31,6 +31,8 @@ func (t tss) IsTombstoneAvailable(ctx context.Context, _ oid.Address, epoch uint } func TestLockUserScenario(t *testing.T) { + t.Parallel() + // Tested user actions: // 1. stores some object // 2. locks the object @@ -146,6 +148,8 @@ func TestLockUserScenario(t *testing.T) { } func TestLockExpiration(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked @@ -222,6 +226,8 @@ func TestLockExpiration(t *testing.T) { } func TestLockForceRemoval(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index c53e03bbf..8131fcf0d 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -17,6 +17,8 @@ import ( ) func TestRebalance(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) const ( @@ -101,6 +103,8 @@ loop: } func TestRebalanceSingleThread(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) obj := testutil.GenerateObjectWithCID(cidtest.ID()) -- 2.45.2 From fe4082799a106a720240f685d3f5c5d84b64fabb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:44:02 +0300 Subject: [PATCH 014/233] [#321] adm: Create multisig accounts in parallel Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/generate.go | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index e714482dd..8975a6d76 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -21,6 +21,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" ) const ( @@ -92,28 +93,30 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er pubs[i] = w.Accounts[0].PrivateKey().PublicKey() } + var errG errgroup.Group + // Create committee account with N/2+1 multi-signature. majCount := smartcontract.GetMajorityHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, majCount, committeeAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create committee account: %w", err) - } - } - // Create consensus account with 2*N/3+1 multi-signature. bftCount := smartcontract.GetDefaultHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, bftCount, consensusAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create consensus account: %w", err) - } + for i := range wallets { + i := i + errG.Go(func() error { + if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], pubs); err != nil { + return fmt.Errorf("can't create committee account: %w", err) + } + if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], pubs); err != nil { + return fmt.Errorf("can't create consentus account: %w", err) + } + if err := wallets[i].SavePretty(); err != nil { + return fmt.Errorf("can't save wallet: %w", err) + } + return nil + }) } - - for _, w := range wallets { - if err := w.SavePretty(); err != nil { - return nil, fmt.Errorf("can't save wallet: %w", err) - } + if err := errG.Wait(); err != nil { + return nil, err } - return passwords, nil } -- 2.45.2 From c8c5f14e2e4fcc22291307cffa650e7472cb6002 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:45:43 +0300 Subject: [PATCH 015/233] [#321] adm/test: Check wallet correctness in parallel Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/generate_test.go | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 39cfc5718..457813df0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -71,24 +72,31 @@ func TestGenerateAlphabet(t *testing.T) { buf.WriteString(testContractPassword + "\r") require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + var wg sync.WaitGroup for i := uint64(0); i < size; i++ { - p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") - w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") - require.Equal(t, 3, len(w.Accounts), "not all accounts were created") - for _, a := range w.Accounts { - err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) - require.NoError(t, err, "can't decrypt account") - switch a.Label { - case consensusAccountName: - require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) - case committeeAccountName: - require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) - default: - require.Equal(t, singleAccountName, a.Label) + i := i + go func() { + defer wg.Done() + p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") + w, err := wallet.NewWalletFromFile(p) + require.NoError(t, err, "wallet doesn't exist") + require.Equal(t, 3, len(w.Accounts), "not all accounts were created") + + for _, a := range w.Accounts { + err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) + require.NoError(t, err, "can't decrypt account") + switch a.Label { + case consensusAccountName: + require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) + case committeeAccountName: + require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) + default: + require.Equal(t, singleAccountName, a.Label) + } } - } + }() } + wg.Wait() t.Run("check contract group wallet", func(t *testing.T) { p := filepath.Join(walletDir, contractWalletFilename) -- 2.45.2 From c62025c8365126537964df0ac536756400063bbd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 17:02:27 +0300 Subject: [PATCH 016/233] [#321] metabase/test: execute tests in parallel Signed-off-by: Evgenii Stratonikov --- .../metabase/containers_test.go | 6 +++ .../metabase/counter_test.go | 49 ++++++++++--------- .../metabase/generic_test.go | 2 + .../metabase/list_test.go | 4 ++ .../metabase/lock_test.go | 6 +++ .../metabase/select_test.go | 18 +++++++ .../metabase/storage_id_test.go | 2 + 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index a0be2c743..c0565b35a 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -15,6 +15,8 @@ import ( ) func TestDB_Containers(t *testing.T) { + t.Parallel() + db := newDB(t) const N = 10 @@ -90,6 +92,8 @@ func TestDB_Containers(t *testing.T) { } func TestDB_ContainersCount(t *testing.T) { + t.Parallel() + db := newDB(t) const R, T, SG, L = 10, 11, 12, 13 // amount of object per type @@ -133,6 +137,8 @@ func TestDB_ContainersCount(t *testing.T) { } func TestDB_ContainerSize(t *testing.T) { + t.Parallel() + db := newDB(t) const ( diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 17a593b6d..507bfcd89 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -2,6 +2,7 @@ package meta_test import ( "context" + "os" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -16,19 +17,23 @@ import ( const objCount = 10 func TestCounters(t *testing.T) { - db := newDB(t) - - var c meta.ObjectCounters - var err error + t.Parallel() + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(t.Name())) + }) t.Run("defaults", func(t *testing.T) { - c, err = db.ObjectCounters() + t.Parallel() + db := newDB(t) + c, err := db.ObjectCounters() require.NoError(t, err) require.Zero(t, c.Phy()) require.Zero(t, c.Logic()) }) t.Run("put", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := make([]*object.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) @@ -39,10 +44,10 @@ func TestCounters(t *testing.T) { for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) - _, err = db.Put(context.Background(), prm) + _, err := db.Put(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) @@ -50,9 +55,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("delete", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) var prm meta.DeletePrm @@ -63,7 +68,7 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i), c.Phy()) @@ -71,9 +76,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) inhumedObjs := make([]oid.Address, objCount/2) @@ -94,16 +99,16 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) }) - require.NoError(t, db.Reset()) - t.Run("put_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) parObj := testutil.GenerateObject() // put objects and check that parent info @@ -116,16 +121,16 @@ func TestCounters(t *testing.T) { require.NoError(t, putBig(db, o)) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) require.Equal(t, uint64(i+1), c.Logic()) } }) - require.NoError(t, db.Reset()) - t.Run("delete_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) // delete objects that have parent info @@ -141,9 +146,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) inhumedObjs := make([]oid.Address, objCount/2) @@ -160,10 +165,10 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - _, err = db.Inhume(context.Background(), prm) + _, err := db.Inhume(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) diff --git a/pkg/local_object_storage/metabase/generic_test.go b/pkg/local_object_storage/metabase/generic_test.go index 227aa9f8d..9d15b6f7a 100644 --- a/pkg/local_object_storage/metabase/generic_test.go +++ b/pkg/local_object_storage/metabase/generic_test.go @@ -10,6 +10,8 @@ import ( ) func TestGeneric(t *testing.T) { + t.Parallel() + defer func() { _ = os.RemoveAll(t.Name()) }() var n int diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 07a0d80f8..4bf3ca827 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -66,6 +66,8 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { } func TestLisObjectsWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const ( @@ -159,6 +161,8 @@ func TestLisObjectsWithCursor(t *testing.T) { } func TestAddObjectDuringListingWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const total = 5 diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 7b62841dc..d5e063431 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -17,6 +17,8 @@ import ( ) func TestDB_Lock(t *testing.T) { + t.Parallel() + cnr := cidtest.ID() db := newDB(t) @@ -171,6 +173,8 @@ func TestDB_Lock(t *testing.T) { } func TestDB_Lock_Expired(t *testing.T) { + t.Parallel() + es := &epochState{e: 123} db := newDB(t, meta.WithEpochState(es)) @@ -192,6 +196,8 @@ func TestDB_Lock_Expired(t *testing.T) { } func TestDB_IsLocked(t *testing.T) { + t.Parallel() + db := newDB(t) // existing and locked objs diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index dab4c028d..e107085ab 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -20,6 +20,8 @@ import ( ) func TestDB_SelectUserAttributes(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -142,6 +144,8 @@ func TestDB_SelectUserAttributes(t *testing.T) { } func TestDB_SelectRootPhyParent(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -293,6 +297,8 @@ func TestDB_SelectRootPhyParent(t *testing.T) { } func TestDB_SelectInhume(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -325,6 +331,8 @@ func TestDB_SelectInhume(t *testing.T) { } func TestDB_SelectPayloadHash(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -393,6 +401,8 @@ func TestDB_SelectPayloadHash(t *testing.T) { } func TestDB_SelectWithSlowFilters(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -498,6 +508,8 @@ func TestDB_SelectWithSlowFilters(t *testing.T) { } func TestDB_SelectObjectID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -611,6 +623,8 @@ func TestDB_SelectObjectID(t *testing.T) { } func TestDB_SelectSplitID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -665,6 +679,8 @@ func TestDB_SelectSplitID(t *testing.T) { } func TestDB_SelectContainerID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -750,6 +766,8 @@ func BenchmarkSelect(b *testing.B) { } func TestExpiredObjects(t *testing.T) { + t.Parallel() + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 5b27cdc87..b3652a680 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -12,6 +12,8 @@ import ( ) func TestDB_StorageID(t *testing.T) { + t.Parallel() + db := newDB(t) raw1 := testutil.GenerateObject() -- 2.45.2 From 147ae8728a250a4cce9ad8da19a6f8c76a46a33d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 15:39:35 +0300 Subject: [PATCH 017/233] [#309] go.mod: Update paulmach/orb to v0.9.1 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 3 ++- go.sum | Bin 95389 -> 96495 bytes 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58fe9c3ed..07e7b043f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Changelog for FrostFS Node ### Removed ### Updated +- `paulmach/orb` to v0.9.1 ### Updating from v0.36.0 ## [v0.36.0] - 2023-04-12 - Furtwängler diff --git a/go.mod b/go.mod index ae0bc88c1..0193937f3 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/nspcc-dev/neo-go v0.100.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 - github.com/paulmach/orb v0.2.2 + github.com/paulmach/orb v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 @@ -94,6 +94,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.5 // indirect github.com/urfave/cli v1.22.5 // indirect + go.mongodb.org/mongo-driver v1.11.4 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect diff --git a/go.sum b/go.sum index 633ac9a1838b2911ff66612c1ab9a3d6e1694380..68526591d7b90d9bb38251f3bbd49944546896d0 100644 GIT binary patch delta 1178 zcmaKsO{?Qn9LB-%g3Ln2g@_}}WkeK8o1COc8etY`)7v&lo1{r!Zg)ve-rwJwZhF6g zi*CKlClHiwW+9BYcR3%zg@{Xcw&RGsAo7R9S^b{p|D1<&|LfK7e_Vb4ET-`eiP&)` zjN!EdzD96l>7}%6jD4OchZzy{hhY^x(#%ThPnZJhHhTy)4{O*@0d%6S8o@a~znDHP`b|sYIX2SqK z@sgQi42BX=*6UU#SFTv2C%xy7ZMSery<@~7l4xZRYKDWvnSOSA!d*Bb!iBH{TiMES zJ(NQdmxmlf2D4G=m2=Ii8R9(!8Ahd`35WXIFE2w>5<{sQI1{q%Vx+Lux)M=|vMHu? zI4UrBiwzuZz}A?3dzjpRb^P>Rdh&?ufJ6B5_87c;{O;a`wH5?Ol9@%(TvyhMsV09r#xK|_+#X(xtC!l@< zmdSW6FYwst+kPO7Dx)8*dMC&Ge{0mlWvFF7lK4j6yJ&q606<|F4si6;dJ{8olOVbz zywJ?X_1caot_-AB39EcubMbNnZbytnE(N>W&=UeS8<>X7gOr-`@am_l=^2uIJ1#A_cqMj%FN!R2RhwzZMs_HX+DDjT!hJ}5ilO#q)nh|29iy^j1#9DFdVzP zwU(cB&Y!{mGcQ(XNvg6Vni2!LuOjt2@fB({CoB}gv?kKhEF5(a(W_|-vyXQ_|M}6! dB!*!C2LNlM1VIxt4lI*~hi_hs$K-YU#a}aZjJ^N> delta 261 zcmaF=k#+7%)(uMeo9pvG2u!wb6x{ry;jFZgk)Dx4hM`qJQcg&|w_{SML4`+HWkG6% zzI#ekT4s`?p>wKpnxAR0L1iaO8(>(b7U=m+Vs=&^>Xu5K)T(slfz32azY(F z!wMZsGaUWHj8cq?3X<{yD|0ek{ImR%eZnnMLW|N;Ci~1)579NyGcqtTG%z(XGBh(W zH#XEYNi{W2HnT7{H8Dmu*Qhv5yEIBaKQP10EY~E$D=aG4G_x=?%FM{g&%(mB!q8Pa zBqZ1%&!@_B(majL7v}~@Z?4|S!!o&kpU`Hfef3P6Z4T8jZ_YgSMQHQsn=11GZj)Tn -- 2.45.2 From 02c02974b3967d6968b207f13c65ba8adea8f702 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 16:23:58 +0300 Subject: [PATCH 018/233] [#309] locode: Parallelize DB generation For v0.4.0 release: Before: ``` Executed in 571.64 secs fish external usr time 283.07 secs 744.00 micros 283.07 secs sys time 8.41 secs 179.00 micros 8.41 secs ``` After: ``` Executed in 54.23 secs fish external usr time 418.65 secs 1.01 millis 418.65 secs sys time 0.61 secs 0.25 millis 0.60 secs ``` Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/util/locode/db/boltdb/calls.go | 2 +- pkg/util/locode/db/db.go | 130 ++++++++++++++++------------- 3 files changed, 74 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e7b043f..316518e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changelog for FrostFS Node - Reload pprof and metrics on SIGHUP for ir (#125) ### Changed +- `frostfs-cli util locode generate` is now much faster (#309) ### Fixed - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) diff --git a/pkg/util/locode/db/boltdb/calls.go b/pkg/util/locode/db/boltdb/calls.go index 171808af2..6a80def3a 100644 --- a/pkg/util/locode/db/boltdb/calls.go +++ b/pkg/util/locode/db/boltdb/calls.go @@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) { // Must not be called before successful Open call. // Must not be called in read-only mode: behavior is undefined. func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { - return db.bolt.Update(func(tx *bbolt.Tx) error { + return db.bolt.Batch(func(tx *bbolt.Tx) error { countryKey, err := countryBucketKey(key.CountryCode()) if err != nil { return err diff --git a/pkg/util/locode/db/db.go b/pkg/util/locode/db/db.go index 2a0f26689..8c71ea794 100644 --- a/pkg/util/locode/db/db.go +++ b/pkg/util/locode/db/db.go @@ -3,8 +3,10 @@ package locodedb import ( "errors" "fmt" + "runtime" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" + "golang.org/x/sync/errgroup" ) // SourceTable is an interface of the UN/LOCODE table. @@ -75,81 +77,93 @@ type NamesDB interface { // FillDatabase generates the FrostFS location database based on the UN/LOCODE table. func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { - return table.IterateAll(func(tableRecord locode.Record) error { - if tableRecord.LOCODE.LocationCode() == "" { + var errG errgroup.Group + + // Pick some sane default, after this the performance stopped increasing. + errG.SetLimit(runtime.NumCPU() * 4) + _ = table.IterateAll(func(tableRecord locode.Record) error { + errG.Go(func() error { + return processTableRecord(tableRecord, airports, continents, names, db) + }) + return nil + }) + return errG.Wait() +} + +func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { + if tableRecord.LOCODE.LocationCode() == "" { + return nil + } + + dbKey, err := NewKey(tableRecord.LOCODE) + if err != nil { + return err + } + + dbRecord, err := NewRecord(tableRecord) + if err != nil { + if errors.Is(err, errParseCoordinates) { return nil } - dbKey, err := NewKey(tableRecord.LOCODE) - if err != nil { - return err - } + return err + } - dbRecord, err := NewRecord(tableRecord) + geoPoint := dbRecord.GeoPoint() + countryName := "" + + if geoPoint == nil { + airportRecord, err := airports.Get(tableRecord) if err != nil { - if errors.Is(err, errParseCoordinates) { + if errors.Is(err, ErrAirportNotFound) { return nil } return err } - geoPoint := dbRecord.GeoPoint() - countryName := "" + geoPoint = airportRecord.Point + countryName = airportRecord.CountryName + } - if geoPoint == nil { - airportRecord, err := airports.Get(tableRecord) - if err != nil { - if errors.Is(err, ErrAirportNotFound) { - return nil - } + dbRecord.SetGeoPoint(geoPoint) - return err - } - - geoPoint = airportRecord.Point - countryName = airportRecord.CountryName - } - - dbRecord.SetGeoPoint(geoPoint) - - if countryName == "" { - countryName, err = names.CountryName(dbKey.CountryCode()) - if err != nil { - if errors.Is(err, ErrCountryNotFound) { - return nil - } - - return err - } - } - - dbRecord.SetCountryName(countryName) - - if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { - subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) - if err != nil { - if errors.Is(err, ErrSubDivNotFound) { - return nil - } - - return err - } - - dbRecord.SetSubDivName(subDivName) - } - - continent, err := continents.PointContinent(geoPoint) + if countryName == "" { + countryName, err = names.CountryName(dbKey.CountryCode()) if err != nil { - return fmt.Errorf("could not calculate continent geo point: %w", err) - } else if continent.Is(ContinentUnknown) { - return nil + if errors.Is(err, ErrCountryNotFound) { + return nil + } + + return err + } + } + + dbRecord.SetCountryName(countryName) + + if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { + subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) + if err != nil { + if errors.Is(err, ErrSubDivNotFound) { + return nil + } + + return err } - dbRecord.SetContinent(continent) + dbRecord.SetSubDivName(subDivName) + } - return db.Put(*dbKey, *dbRecord) - }) + continent, err := continents.PointContinent(geoPoint) + if err != nil { + return fmt.Errorf("could not calculate continent geo point: %w", err) + } else if continent.Is(ContinentUnknown) { + return nil + } + + dbRecord.SetContinent(continent) + + return db.Put(*dbKey, *dbRecord) } // LocodeRecord returns the record from the FrostFS location database -- 2.45.2 From 6055b183620da920d821053f69b02e7dc55e4215 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 13:24:08 +0300 Subject: [PATCH 019/233] [#342] morph: Remove unused toStackParameter() Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 59 --------------------------------- pkg/morph/client/client_test.go | 57 ------------------------------- 2 files changed, 116 deletions(-) delete mode 100644 pkg/morph/client/client_test.go diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 284e065fb..c9ff14a66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -21,7 +21,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -444,64 +443,6 @@ func (c *Client) roleList(r noderoles.Role) (keys.PublicKeys, error) { return c.rolemgmt.GetDesignatedByRole(r, height) } -// tries to resolve sc.Parameter from the arg. -// -// Wraps any error to frostfsError. -func toStackParameter(value any) (sc.Parameter, error) { - var res = sc.Parameter{ - Value: value, - } - - switch v := value.(type) { - case []byte: - res.Type = sc.ByteArrayType - case int: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case int64: - res.Type = sc.IntegerType - res.Value = big.NewInt(v) - case uint64: - res.Type = sc.IntegerType - res.Value = new(big.Int).SetUint64(v) - case [][]byte: - arr := make([]sc.Parameter, 0, len(v)) - for i := range v { - elem, err := toStackParameter(v[i]) - if err != nil { - return res, err - } - - arr = append(arr, elem) - } - - res.Type = sc.ArrayType - res.Value = arr - case string: - res.Type = sc.StringType - case util.Uint160: - res.Type = sc.ByteArrayType - res.Value = v.BytesBE() - case noderoles.Role: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case keys.PublicKeys: - arr := make([][]byte, 0, len(v)) - for i := range v { - arr = append(arr, v[i].Bytes()) - } - - return toStackParameter(arr) - case bool: - res.Type = sc.BoolType - res.Value = v - default: - return res, wrapFrostFSError(fmt.Errorf("chain/client: unsupported parameter %v", value)) - } - - return res, nil -} - // MagicNumber returns the magic number of the network // to which the underlying RPC node client is connected. func (c *Client) MagicNumber() (uint64, error) { diff --git a/pkg/morph/client/client_test.go b/pkg/morph/client/client_test.go deleted file mode 100644 index a448c2cf4..000000000 --- a/pkg/morph/client/client_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package client - -import ( - "math/big" - "testing" - - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/stretchr/testify/require" -) - -func TestToStackParameter(t *testing.T) { - items := []struct { - value any - expType sc.ParamType - expVal any - }{ - { - value: []byte{1, 2, 3}, - expType: sc.ByteArrayType, - }, - { - value: int64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: uint64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: "hello world", - expType: sc.StringType, - }, - { - value: false, - expType: sc.BoolType, - }, - { - value: true, - expType: sc.BoolType, - }, - } - - for _, item := range items { - t.Run(item.expType.String()+" to stack parameter", func(t *testing.T) { - res, err := toStackParameter(item.value) - require.NoError(t, err) - require.Equal(t, item.expType, res.Type) - if item.expVal != nil { - require.Equal(t, item.expVal, res.Value) - } else { - require.Equal(t, item.value, res.Value) - } - }) - } -} -- 2.45.2 From a6ee7a3087b162b2c67e9f3e2a8d1580a86b59bf Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 12 May 2023 11:44:21 +0300 Subject: [PATCH 020/233] [#324] Add replicator metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/object.go | 1 + pkg/metrics/node.go | 5 +++ pkg/metrics/replicator.go | 59 +++++++++++++++++++++++++++ pkg/services/replicator/metrics.go | 8 ++++ pkg/services/replicator/process.go | 5 +++ pkg/services/replicator/replicator.go | 8 ++++ 6 files changed, 86 insertions(+) create mode 100644 pkg/metrics/replicator.go create mode 100644 pkg/services/replicator/metrics.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 4ff9b8522..1ce330cb5 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -267,6 +267,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), + replicator.WithMetrics(c.metricsCollector), ) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0f9c6183d..bf12e610f 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,6 +8,7 @@ type NodeMetrics struct { objectServiceMetrics engineMetrics stateMetrics + replicatorMetrics epoch metric[prometheus.Gauge] } @@ -21,6 +22,9 @@ func NewNodeMetrics() *NodeMetrics { state := newStateMetrics() state.register() + replicator := newReplicatorMetrics() + replicator.register() + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, @@ -33,6 +37,7 @@ func NewNodeMetrics() *NodeMetrics { objectServiceMetrics: objectService, engineMetrics: engine, stateMetrics: state, + replicatorMetrics: replicator, epoch: epoch, } } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go new file mode 100644 index 000000000..55f736c66 --- /dev/null +++ b/pkg/metrics/replicator.go @@ -0,0 +1,59 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const replicatorSubsystem = "replicator" + +type replicatorMetrics struct { + inFlightRequests metric[prometheus.Gauge] + processedObjects metric[prometheus.Counter] + totalReplicatedPayloadSize metric[prometheus.Counter] +} + +func (m replicatorMetrics) IncInFlightRequest() { + m.inFlightRequests.value.Inc() +} + +func (m replicatorMetrics) DecInFlightRequest() { + m.inFlightRequests.value.Dec() +} + +func (m replicatorMetrics) IncProcessedObjects() { + m.processedObjects.value.Inc() +} + +func (m replicatorMetrics) AddPayloadSize(size int64) { + m.totalReplicatedPayloadSize.value.Add(float64(size)) +} + +func newReplicatorMetrics() replicatorMetrics { + return replicatorMetrics{ + inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), + processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), + totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), + } +} + +func (m replicatorMetrics) register() { + mustRegister(m.inFlightRequests) + mustRegister(m.processedObjects) + mustRegister(m.totalReplicatedPayloadSize) +} + +func newReplicatorCounter(name, help string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} + +func newReplicatorGauge(name, help string) metric[prometheus.Gauge] { + return newGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} diff --git a/pkg/services/replicator/metrics.go b/pkg/services/replicator/metrics.go new file mode 100644 index 000000000..3fc062926 --- /dev/null +++ b/pkg/services/replicator/metrics.go @@ -0,0 +1,8 @@ +package replicator + +type MetricsRegister interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 46e0c9468..0f82ff232 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -20,6 +20,8 @@ type TaskResult interface { // HandleTask executes replication task inside invoking goroutine. // Passes all the nodes that accepted the replication to the TaskResult. func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) { + p.metrics.IncInFlightRequest() + defer p.metrics.DecInFlightRequest() defer func() { p.log.Debug(logs.ReplicatorFinishWork, zap.Uint32("amount of unfinished replicas", task.quantity), @@ -69,6 +71,9 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) task.quantity-- res.SubmitSuccessfulReplication(task.nodes[i]) + + p.metrics.IncProcessedObjects() + p.metrics.AddPayloadSize(int64(task.obj.PayloadSize())) } } } diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index 493982100..bb817cb32 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -26,6 +26,8 @@ type cfg struct { remoteSender *putsvc.RemoteSender localStorage *engine.StorageEngine + + metrics MetricsRegister } func defaultCfg() *cfg { @@ -74,3 +76,9 @@ func WithLocalStorage(v *engine.StorageEngine) Option { c.localStorage = v } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} -- 2.45.2 From 14c35d776edcbe5c069052ad3d7bb3f4ecd845fc Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 1 Feb 2023 10:21:17 +0300 Subject: [PATCH 021/233] [#39] node: Add optional profilers Include settings for block and mutex profilers. They are disabled by default, as in Go runtime itself. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/profiler/config.go | 24 ++++++++++++++++++ .../config/profiler/config_test.go | 6 +++++ cmd/frostfs-node/main.go | 3 +-- cmd/frostfs-node/pprof.go | 25 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 4 ++- config/example/node.yaml | 2 ++ docs/storage-node-configuration.md | 20 +++++++++++---- 8 files changed, 78 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/config/profiler/config.go b/cmd/frostfs-node/config/profiler/config.go index f891833a6..191694970 100644 --- a/cmd/frostfs-node/config/profiler/config.go +++ b/cmd/frostfs-node/config/profiler/config.go @@ -51,3 +51,27 @@ func Address(c *config.Config) string { return AddressDefault } + +// BlockRates returns the value of "block_rate" config parameter +// from "pprof" section. +func BlockRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "block_rate")) + if v <= 0 { + return 0 + } + return v +} + +// MutexRate returns the value of "mutex_rate" config parameter +// from "pprof" section. +func MutexRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "mutex_rate")) + if v <= 0 { + return 0 + } + return v +} diff --git a/cmd/frostfs-node/config/profiler/config_test.go b/cmd/frostfs-node/config/profiler/config_test.go index bb1b20eb8..355874387 100644 --- a/cmd/frostfs-node/config/profiler/config_test.go +++ b/cmd/frostfs-node/config/profiler/config_test.go @@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to) require.Equal(t, profilerconfig.AddressDefault, addr) require.False(t, profilerconfig.Enabled(configtest.EmptyConfig())) + + require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig())) + require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" @@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, 15*time.Second, to) require.Equal(t, "localhost:6060", addr) require.True(t, profilerconfig.Enabled(c)) + + require.Equal(t, 10_000, profilerconfig.BlockRate(c)) + require.Equal(t, 10_000, profilerconfig.MutexRate(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 4fcc38e08..a2024706b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -83,9 +83,8 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() - pprof, _ := pprofComponent(c) metrics, _ := metricsComponent(c) - initAndLog(c, pprof.name, pprof.init) + initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 9be2dd9df..dcd320146 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -1,10 +1,19 @@ package main import ( + "runtime" + profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) +func initProfilerService(c *cfg) { + tuneProfilers(c) + + pprof, _ := pprofComponent(c) + pprof.init(c) +} + func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before @@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.pprof.cfg = c c.dynamicConfiguration.pprof.name = "pprof" c.dynamicConfiguration.pprof.handler = httputil.Handler() + c.dynamicConfiguration.pprof.preReload = tuneProfilers updated = true } @@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { return c.dynamicConfiguration.pprof, updated } + +func tuneProfilers(c *cfg) { + // Disabled by default, see documentation for + // runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction(). + blockRate := 0 + mutexRate := 0 + + if profilerconfig.Enabled(c.appCfg) { + blockRate = profilerconfig.BlockRate(c.appCfg) + mutexRate = profilerconfig.MutexRate(c.appCfg) + } + + runtime.SetBlockProfileRate(blockRate) + runtime.SetMutexProfileFraction(mutexRate) +} diff --git a/config/example/node.env b/config/example/node.env index 77992d995..143bf0388 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s +FROSTFS_PPROF_BLOCK_RATE=10000 +FROSTFS_PPROF_MUTEX_RATE=10000 FROSTFS_PROMETHEUS_ENABLED=true FROSTFS_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/node.json b/config/example/node.json index b52eb6d96..04aabdd42 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -5,7 +5,9 @@ "pprof": { "enabled": true, "address": "localhost:6060", - "shutdown_timeout": "15s" + "shutdown_timeout": "15s", + "block_rate": 10000, + "mutex_rate": 10000 }, "prometheus": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 1669e0e86..bc665a688 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -5,6 +5,8 @@ pprof: enabled: true address: localhost:6060 # endpoint for Node profiling shutdown_timeout: 15s # timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 366c263a0..2c78cf6b1 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -75,13 +75,23 @@ element. Contains configuration for the `pprof` profiler. -| Parameter | Type | Default value | Description | -|--------------------|------------|---------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | | Address that service listener binds to. | -| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| Parameter | Type | Default value | Description | +|--------------------|-----------------------------------|---------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | | Address that service listener binds to. | +| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| `debug` | [Debug config](#debug-subsection) | | Optional profiles configuration | +## `debug` subsection + +Contains optional profiles configuration. + +| Parameter | Type | Default value | Description | +|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `block_rate` | `int` | `0` | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate. | +| `mutex_rate` | `int` | `0` | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. | + # `prometheus` section Contains configuration for the `prometheus` metrics service. -- 2.45.2 From 61776033c2df233c520fe55783058ee724c936a1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 10:58:52 +0300 Subject: [PATCH 022/233] [#39] ir: Do not reload services if they are disabled Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index d3cda8930..d73160a1d 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -66,7 +66,7 @@ func (c *httpComponent) reload() { enabled := cfg.GetBool(c.enabledKey) address := cfg.GetString(c.addressKey) dur := cfg.GetDuration(c.shutdownTimeoutKey) - if enabled != c.enabled || address != c.address || dur != c.shutdownDur { + if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, -- 2.45.2 From f989bc52be968a4ce7f96adb4da0bd550dae33f1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 11:04:52 +0300 Subject: [PATCH 023/233] [#39] ir: Do not store config keys in `httpComponent` Pprof will have specific options, it seems wrong to have them in a generic struct. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 33 ++++++++++++++++++--------------- cmd/frostfs-ir/metrics.go | 13 ++----------- cmd/frostfs-ir/pprof.go | 13 ++----------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index d73160a1d..38e936471 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -11,22 +11,25 @@ import ( ) type httpComponent struct { - srv *httputil.Server - address string - addressKey string - name string - handler http.Handler - shutdownDur time.Duration - shutdownTimeoutKey string - enabled bool - enabledKey string + srv *httputil.Server + address string + name string + handler http.Handler + shutdownDur time.Duration + enabled bool } +const ( + enabledKeyPostfix = ".enabled" + addressKeyPostfix = ".address" + shutdownTimeoutKeyPostfix = ".shutdown_timeout" +) + func (c *httpComponent) init() { log.Info(fmt.Sprintf("init %s", c.name)) - c.enabled = cfg.GetBool(c.enabledKey) - c.address = cfg.GetString(c.addressKey) - c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey) + c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) + c.address = cfg.GetString(c.name + addressKeyPostfix) + c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) if c.enabled { c.srv = httputil.New( @@ -63,9 +66,9 @@ func (c *httpComponent) shutdown() error { func (c *httpComponent) reload() { log.Info(fmt.Sprintf("reload %s", c.name)) - enabled := cfg.GetBool(c.enabledKey) - address := cfg.GetString(c.addressKey) - dur := cfg.GetDuration(c.shutdownTimeoutKey) + enabled := cfg.GetBool(c.name + enabledKeyPostfix) + address := cfg.GetString(c.name + addressKeyPostfix) + dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go index debeb7d60..39b432c74 100644 --- a/cmd/frostfs-ir/metrics.go +++ b/cmd/frostfs-ir/metrics.go @@ -4,18 +4,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) -const ( - prometheusEnabledKey = "prometheus.enabled" - prometheusAddressKey = "prometheus.address" - prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout" -) - func newMetricsComponent() *httpComponent { return &httpComponent{ - name: "prometheus", - enabledKey: prometheusEnabledKey, - addressKey: prometheusAddressKey, - shutdownTimeoutKey: prometheusShutdownTimeoutKey, - handler: metrics.Handler(), + name: "prometheus", + handler: metrics.Handler(), } } diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 0d48c11de..8228a0f53 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -4,18 +4,9 @@ import ( httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) -const ( - pprofEnabledKey = "pprof.enabled" - pprofAddressKey = "pprof.address" - pprofShutdownTimeoutKey = "pprof.shutdown_timeout" -) - func newPprofComponent() *httpComponent { return &httpComponent{ - name: "pprof", - enabledKey: pprofEnabledKey, - addressKey: pprofAddressKey, - shutdownTimeoutKey: pprofShutdownTimeoutKey, - handler: httputil.Handler(), + name: "pprof", + handler: httputil.Handler(), } } -- 2.45.2 From f604d6bbdce327a3560020864b9f37857d0ebbf0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 19 Apr 2023 20:02:02 +0300 Subject: [PATCH 024/233] [#39] ir: Add optional profilers Includes `block` and `mutex` profiles configuration. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 10 ++++-- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-ir/pprof.go | 64 ++++++++++++++++++++++++++++++--- config/example/ir.env | 2 ++ config/example/ir.yaml | 2 ++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 38e936471..8f5e3753b 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -64,12 +64,16 @@ func (c *httpComponent) shutdown() error { return nil } -func (c *httpComponent) reload() { - log.Info(fmt.Sprintf("reload %s", c.name)) +func (c *httpComponent) needReload() bool { enabled := cfg.GetBool(c.name + enabledKeyPostfix) address := cfg.GetString(c.name + addressKeyPostfix) dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) - if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { + return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 84dff9101..1718a46ab 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -30,7 +30,7 @@ var ( intErr = make(chan error) // internal inner ring errors logPrm = new(logger.Prm) innerRing *innerring.Server - pprofCmp *httpComponent + pprofCmp *pprofComponent metricsCmp *httpComponent log *logger.Logger cfg *viper.Viper diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 8228a0f53..d67c463fc 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -1,12 +1,68 @@ package main import ( + "fmt" + "runtime" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" ) -func newPprofComponent() *httpComponent { - return &httpComponent{ - name: "pprof", - handler: httputil.Handler(), +type pprofComponent struct { + httpComponent + blockRate int + mutexRate int +} + +const ( + pprofBlockRateKey = "pprof.block_rate" + pprofMutexRateKey = "pprof.mutex_rate" +) + +func newPprofComponent() *pprofComponent { + return &pprofComponent{ + httpComponent: httpComponent{ + name: "pprof", + handler: httputil.Handler(), + }, + } +} + +func (c *pprofComponent) init() { + c.httpComponent.init() + + if c.enabled { + c.blockRate = cfg.GetInt(pprofBlockRateKey) + c.mutexRate = cfg.GetInt(pprofMutexRateKey) + runtime.SetBlockProfileRate(c.blockRate) + runtime.SetMutexProfileFraction(c.mutexRate) + } else { + c.blockRate = 0 + c.mutexRate = 0 + runtime.SetBlockProfileRate(0) + runtime.SetMutexProfileFraction(0) + } +} + +func (c *pprofComponent) needReload() bool { + blockRate := cfg.GetInt(pprofBlockRateKey) + mutexRate := cfg.GetInt(pprofMutexRateKey) + return c.httpComponent.needReload() || + c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate) +} + +func (c *pprofComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error())) + return + } + + c.init() + c.start() } } diff --git a/config/example/ir.env b/config/example/ir.env index e3de23ac1..7b8f8a89d 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -74,6 +74,8 @@ FROSTFS_IR_CONTRACTS_ALPHABET_ZHIVETE=f584699bc2ff457d339fb09f16217042c1a42101 FROSTFS_IR_PPROF_ENABLED=true FROSTFS_IR_PPROF_ADDRESS=localhost:6060 FROSTFS_IR_PPROF_SHUTDOWN_TIMEOUT=30s +FROSTFS_IR_PPROF_BLOCK_RATE=10000 +FROSTFS_IR_PPROF_MUTEX_RATE=10000 FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index bd56ec74b..1130a840b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -113,6 +113,8 @@ pprof: enabled: true address: localhost:6060 # Endpoint for application pprof profiling; disabled by default shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true -- 2.45.2 From 429a87e83bf98d4e3a3df8a4977e03856ed30450 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 11:38:43 +0300 Subject: [PATCH 025/233] [#39] ir: Use defer for wg.Done() Not important, but `exitOnErr` can alter control flow, let's be explicit. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 8f5e3753b..3a6d77d84 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -50,8 +50,8 @@ func (c *httpComponent) start() { log.Info(fmt.Sprintf("start %s", c.name)) wg.Add(1) go func() { + defer wg.Done() exitErr(c.srv.Serve()) - wg.Done() }() } } -- 2.45.2 From d4d921dcaf389303844e1e19413e969539c9cadd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 15 May 2023 11:50:20 +0300 Subject: [PATCH 026/233] [#346] adm: fix race in wallet generation Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 8975a6d76..dd327a4b1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -101,11 +101,13 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er bftCount := smartcontract.GetDefaultHonestNodeCount(size) for i := range wallets { i := i + ps := make(keys.PublicKeys, len(pubs)) + copy(ps, pubs) errG.Go(func() error { - if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], pubs); err != nil { + if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) } - if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], pubs); err != nil { + if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create consentus account: %w", err) } if err := wallets[i].SavePretty(); err != nil { -- 2.45.2 From 079b28fa0f51fe918c7857bdbef4ba9fb5d91cc4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 15 May 2023 14:46:17 +0300 Subject: [PATCH 027/233] [#341] Register candidates in separate transactions Signed-off-by: Alejandro Lopez --- .../modules/morph/initialize_register.go | 72 ++++++++++++------- .../internal/modules/morph/initialize_test.go | 6 ++ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index b1542cc92..469b269de 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -19,33 +19,24 @@ import ( ) // initialAlphabetNEOAmount represents the total amount of GAS distributed between alphabet nodes. -const initialAlphabetNEOAmount = native.NEOTotalSupply - -func (c *initializeContext) registerCandidates() error { - neoHash := neo.Hash - - cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neoHash, "getCandidates")) - if err != nil { - return fmt.Errorf("`getCandidates`: %w", err) - } - - if len(cc) > 0 { - c.Command.Println("Candidates are already registered.") - return nil - } +const ( + initialAlphabetNEOAmount = native.NEOTotalSupply + registerBatchSize = transaction.MaxAttributes - 1 +) +func (c *initializeContext) registerCandidateRange(start, end int) error { regPrice, err := c.getCandidateRegisterPrice() if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) } w := io.NewBufBinWriter() - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, 1) - for _, acc := range c.Accounts { - emit.AppCall(w.BinWriter, neoHash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, 1) + for _, acc := range c.Accounts[start:end] { + emit.AppCall(w.BinWriter, neo.Hash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) emit.Opcodes(w.BinWriter, opcode.ASSERT) } - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, regPrice) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, regPrice) if w.Err != nil { panic(fmt.Sprintf("BUG: %v", w.Err)) } @@ -54,14 +45,14 @@ func (c *initializeContext) registerCandidates() error { Signer: c.getSigner(false, c.CommitteeAcc), Account: c.CommitteeAcc, }} - for i := range c.Accounts { + for _, acc := range c.Accounts[start:end] { signers = append(signers, rpcclient.SignerAccount{ Signer: transaction.Signer{ - Account: c.Accounts[i].Contract.ScriptHash(), + Account: acc.Contract.ScriptHash(), Scopes: transaction.CustomContracts, - AllowedContracts: []util.Uint160{neoHash}, + AllowedContracts: []util.Uint160{neo.Hash}, }, - Account: c.Accounts[i], + Account: acc, }) } @@ -74,8 +65,8 @@ func (c *initializeContext) registerCandidates() error { } network := c.CommitteeAct.GetNetwork() - for i := range c.Accounts { - if err := c.Accounts[i].SignTx(network, tx); err != nil { + for _, acc := range c.Accounts[start:end] { + if err := acc.SignTx(network, tx); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } } @@ -83,6 +74,39 @@ func (c *initializeContext) registerCandidates() error { return c.sendTx(tx, c.Command, true) } +func (c *initializeContext) registerCandidates() error { + cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) + if err != nil { + return fmt.Errorf("`getCandidates`: %w", err) + } + + need := len(c.Accounts) + have := len(cc) + + if need == have { + c.Command.Println("Candidates are already registered.") + return nil + } + + // Register candidates in batches in order to overcome the signers amount limit. + // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 + for i := 0; i < need; i += registerBatchSize { + start, end := i, i+registerBatchSize + if end > need { + end = need + } + // This check is sound because transactions are accepted/rejected atomically. + if have >= end { + continue + } + if err := c.registerCandidateRange(start, end); err != nil { + return fmt.Errorf("registering candidates %d..%d: %q", start, end-1, err) + } + } + + return nil +} + func (c *initializeContext) transferNEOToAlphabetContracts() error { neoHash := neo.Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index fb2dc3e3f..e2e5aa0ad 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -37,6 +37,12 @@ func TestInitialize(t *testing.T) { t.Run("7 nodes", func(t *testing.T) { testInitialize(t, 7) }) + t.Run("16 nodes", func(t *testing.T) { + testInitialize(t, 16) + }) + t.Run("22 nodes", func(t *testing.T) { + testInitialize(t, 22) + }) } func testInitialize(t *testing.T, committeeSize int) { -- 2.45.2 From 8da6530f412d73f5ff3641782e889f47052fb54d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 15 May 2023 15:24:29 +0300 Subject: [PATCH 028/233] [#351] cli: Support copies number parameter in `object put` Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-cli/internal/client/client.go | 9 +++++++++ cmd/frostfs-cli/modules/object/put.go | 15 +++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 316518e60..e87a2ca0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Changelog for FrostFS Node - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) - Reload pprof and metrics on SIGHUP for ir (#125) +- Support copies number parameter in `frostfs-cli object put` (#351) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index cbf19eb4b..875ccf904 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -329,6 +329,8 @@ func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { type PutObjectPrm struct { commonObjectPrm + copyNum []uint32 + hdr *object.Object rdr io.Reader @@ -352,6 +354,12 @@ func (x *PutObjectPrm) SetHeaderCallback(f func(*object.Object)) { x.headerCallback = f } +// SetCopiesNumberByVectors sets ordered list of minimal required object copies numbers +// per placement vector. +func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) { + x.copyNum = copiesNumbers +} + // PutObjectRes groups the resulting values of PutObject operation. type PutObjectRes struct { id oid.ID @@ -381,6 +389,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { } putPrm.WithXHeaders(prm.xHeaders...) + putPrm.SetCopiesNumberByVectors(prm.copyNum) wrt, err := prm.cli.ObjectPutInit(context.Background(), putPrm) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index fe8e9dda9..6fd91ca4b 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -25,6 +25,7 @@ import ( const ( noProgressFlag = "no-progress" notificationFlag = "notify" + copiesNumberFlag = "copies-number" ) var putExpiredOn uint64 @@ -56,6 +57,8 @@ func initObjectPutCmd() { flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") + + flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call") } func putObject(cmd *cobra.Command, _ []string) { @@ -116,6 +119,18 @@ func putObject(cmd *cobra.Command, _ []string) { } } + copyNum, err := cmd.Flags().GetString(copiesNumberFlag) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + if len(copyNum) > 0 { + var cn []uint32 + for _, num := range strings.Split(copyNum, ",") { + val, err := strconv.ParseUint(num, 10, 32) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + cn = append(cn, uint32(val)) + } + prm.SetCopiesNumberByVectors(cn) + } + res, err := internalclient.PutObject(prm) if p != nil { p.Finish() -- 2.45.2 From ab07bad33de15a1fadc9ffad15ca8989e8fc5a63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 May 2023 11:43:44 +0300 Subject: [PATCH 029/233] [#332] gc: Add complex object unit test Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc_test.go | 195 +++++++++++++++++----- 1 file changed, 150 insertions(+), 45 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index acc039cd2..0a494fd52 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -2,6 +2,8 @@ package shard_test import ( "context" + "encoding/binary" + "errors" "path/filepath" "testing" "time" @@ -19,62 +21,20 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" ) -func Test_GCDropsLockedExpiredObject(t *testing.T) { +func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { t.Parallel() - var sh *shard.Shard - epoch := &epochState{ Value: 100, } - rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(2), - blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= 1<<20 - }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), - }, - }), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epoch), - ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) - }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { - sh.HandleExpiredLocks(ctx, epoch, a) - }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - } - - sh = shard.New(opts...) - require.NoError(t, sh.Open()) - require.NoError(t, sh.Init(context.Background())) + sh := createAndInitGCTestShard(t, epoch) t.Cleanup(func() { releaseShard(sh, t) @@ -122,3 +82,148 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired object must be deleted") } + +func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { + t.Parallel() + + epoch := &epochState{ + Value: 100, + } + + cnr := cidtest.ID() + parentID := oidtest.ID() + splitID := objectSDK.NewSplitID() + + var objExpirationAttr objectSDK.Attribute + objExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + objExpirationAttr.SetValue("101") + + var lockExpirationAttr objectSDK.Attribute + lockExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + lockExpirationAttr.SetValue("103") + + parent := testutil.GenerateObjectWithCID(cnr) + parent.SetID(parentID) + parent.SetPayload(nil) + parent.SetAttributes(objExpirationAttr) + + const childCount = 10 + children := make([]*objectSDK.Object, childCount) + childIDs := make([]oid.ID, childCount) + for i := range children { + children[i] = testutil.GenerateObjectWithCID(cnr) + if i != 0 { + children[i].SetPreviousID(childIDs[i-1]) + } + if i == len(children)-1 { + children[i].SetParent(parent) + } + children[i].SetSplitID(splitID) + children[i].SetPayload([]byte{byte(i), byte(i + 1), byte(i + 2)}) + childIDs[i], _ = children[i].ID() + } + + link := testutil.GenerateObjectWithCID(cnr) + link.SetParent(parent) + link.SetParentID(parentID) + link.SetSplitID(splitID) + link.SetChildren(childIDs...) + + linkID, _ := link.ID() + + sh := createAndInitGCTestShard(t, epoch) + + t.Cleanup(func() { + releaseShard(sh, t) + }) + + lock := testutil.GenerateObjectWithCID(cnr) + lock.SetType(objectSDK.TypeLock) + lock.SetAttributes(lockExpirationAttr) + lockID, _ := lock.ID() + + var putPrm shard.PutPrm + + for _, child := range children { + putPrm.SetObject(child) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + } + + putPrm.SetObject(link) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + err = sh.Lock(context.Background(), cnr, lockID, append(childIDs, parentID, linkID)) + require.NoError(t, err) + + putPrm.SetObject(lock) + _, err = sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm shard.GetPrm + getPrm.SetAddress(objectCore.AddressOf(parent)) + + _, err = sh.Get(context.Background(), getPrm) + var splitInfoError *objectSDK.SplitInfoError + require.True(t, errors.As(err, &splitInfoError), "split info must be provided") + + epoch.Value = 105 + sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + + require.Eventually(t, func() bool { + _, err = sh.Get(context.Background(), getPrm) + return shard.IsErrNotFound(err) + }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") +} + +func createAndInitGCTestShard(t *testing.T, epoch *epochState) *shard.Shard { + var sh *shard.Shard + + rootPath := t.TempDir() + opts := []shard.Option{ + shard.WithID(shard.NewIDFromBytes(binary.AppendVarint([]byte{}, int64(48)))), + shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), + shard.WithBlobStorOptions( + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(2), + blobovniczatree.WithBlobovniczaShallowWidth(2)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + ), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epoch), + ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + + return pool + }), + shard.WithGCRemoverSleepInterval(1 * time.Millisecond), + } + + sh = shard.New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init(context.Background())) + + return sh +} -- 2.45.2 From 869fcbf59166732d70517ec731fe8ed3c9c9ff5c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 May 2023 12:06:38 +0300 Subject: [PATCH 030/233] [#332] gc: Fix expired complex object deletion Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + internal/logs/logs.go | 1 + pkg/local_object_storage/metabase/children.go | 57 +++++++++++++++++ pkg/local_object_storage/shard/gc.go | 20 ++++++ pkg/local_object_storage/shard/gc_test.go | 64 +------------------ pkg/local_object_storage/shard/range_test.go | 3 +- pkg/local_object_storage/shard/shard_test.go | 27 ++++++-- .../shard/shutdown_test.go | 4 +- 8 files changed, 108 insertions(+), 69 deletions(-) create mode 100644 pkg/local_object_storage/metabase/children.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e87a2ca0c..6a4c48097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Changelog for FrostFS Node - Notary requests parsing according to `neo-go`'s updates (#268) - Tree service panic in its internal client cache (#322) - Iterate over endpoints when create ws client in morph's constructor (#304) +- Delete complex objects with GC (#332) ### Removed ### Updated diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 936042d2d..5cd1b8dba 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -483,4 +483,5 @@ const ( ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" ShardGCRemoveGarbageStarted = "garbage remove started" ShardGCRemoveGarbageCompleted = "garbage remove completed" + ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ) diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go new file mode 100644 index 000000000..f0591b43c --- /dev/null +++ b/pkg/local_object_storage/metabase/children.go @@ -0,0 +1,57 @@ +package meta + +import ( + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" +) + +// GetChildren returns parent -> children map. +// If an object has no children, then map will contain addr -> empty slice value. +func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Address, error) { + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + result := make(map[oid.Address][]oid.Address, len(addresses)) + + buffer := make([]byte, bucketKeySize) + err := db.boltDB.View(func(tx *bbolt.Tx) error { + for _, addr := range addresses { + if _, found := result[addr]; found { + continue + } + + result[addr] = []oid.Address{} + bkt := tx.Bucket(parentBucketName(addr.Container(), buffer)) + if bkt == nil { + continue + } + + binObjIDs, err := decodeList(bkt.Get(objectKey(addr.Object(), buffer))) + if err != nil { + return err + } + + for _, binObjID := range binObjIDs { + var id oid.ID + if err = id.Decode(binObjID); err != nil { + return err + } + var resultAddress oid.Address + resultAddress.SetContainer(addr.Container()) + resultAddress.SetObject(id) + result[addr] = append(result[addr], resultAddress) + } + } + return nil + }) + + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 82876e675..f4f7a21e2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -320,6 +320,12 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + expired, err := s.getExpiredWithLinked(expired) + if err != nil { + s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) + return + } + var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(expired...) @@ -345,6 +351,20 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } } +func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error) { + result := make([]oid.Address, 0, len(source)) + parentToChildren, err := s.metaBase.GetChildren(source) + if err != nil { + return nil, err + } + for parent, children := range parentToChildren { + result = append(result, parent) + result = append(result, children...) + } + + return result, nil +} + func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 0a494fd52..263a0ea4d 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -2,29 +2,20 @@ package shard_test import ( "context" - "encoding/binary" "errors" - "path/filepath" "testing" "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { @@ -34,7 +25,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { Value: 100, } - sh := createAndInitGCTestShard(t, epoch) + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -131,7 +122,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { linkID, _ := link.ID() - sh := createAndInitGCTestShard(t, epoch) + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -176,54 +167,3 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") } - -func createAndInitGCTestShard(t *testing.T, epoch *epochState) *shard.Shard { - var sh *shard.Shard - - rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes(binary.AppendVarint([]byte{}, int64(48)))), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(2), - blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= 1<<20 - }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), - }, - }), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epoch), - ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) - }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { - sh.HandleExpiredLocks(ctx, epoch, a) - }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - shard.WithGCRemoverSleepInterval(1 * time.Millisecond), - } - - sh = shard.New(opts...) - - require.NoError(t, sh.Open()) - require.NoError(t, sh.Init(context.Background())) - - return sh -} diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 4574ce415..9ef2106b0 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -87,7 +87,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { Storage: fstree.New( fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), }, - })}) + })}, + nil) defer releaseShard(sh, t) for _, tc := range testCases { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index fea342766..7b2fdb5d1 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -12,8 +13,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -29,11 +33,13 @@ func (s epochState) CurrentEpoch() uint64 { func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { return newCustomShard(t, t.TempDir(), enableWriteCache, + nil, nil, nil) } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option) *shard.Shard { +func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { + var sh *shard.Shard if enableWriteCache { rootPath = filepath.Join(rootPath, "wc") } else { @@ -67,8 +73,9 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epochState{}), + append([]meta.Option{ + meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{})}, + metaOptions...)..., ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), shard.WithWriteCache(enableWriteCache), @@ -77,9 +84,21 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option{writecache.WithPath(filepath.Join(rootPath, "wcache"))}, wcOpts...)..., ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + shard.WithGCRemoverSleepInterval(1 * time.Millisecond), } - sh := shard.New(opts...) + sh = shard.New(opts...) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 76b20d659..5fe9fd7e9 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -40,7 +40,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { writecache.WithSmallObjectSize(smallSize), writecache.WithMaxObjectSize(smallSize * 2)} - sh := newCustomShard(t, dir, true, wcOpts, nil) + sh := newCustomShard(t, dir, true, wcOpts, nil, nil) var errG errgroup.Group for i := range objects { @@ -55,7 +55,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) - sh = newCustomShard(t, dir, true, wcOpts, nil) + sh = newCustomShard(t, dir, true, wcOpts, nil, nil) defer releaseShard(sh, t) var getPrm shard.GetPrm -- 2.45.2 From 0624820909ac344eb0038a2fc3557137e1497af0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 14:49:04 +0300 Subject: [PATCH 031/233] [#336] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 7 +++++++ go.mod | 34 +++++++++++++++++----------------- go.sum | Bin 96495 -> 96389 bytes 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4c48097..c05d065fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,13 @@ Changelog for FrostFS Node ### Removed ### Updated - `paulmach/orb` to v0.9.1 +- `github.com/nats-io/nats.go` to `v1.25.0` +- `golang.org/x/sync` to `v0.2.0` +- `golang.org/x/term` to `v0.8.0` +- `github.com/spf13/cobra` to `v1.7.0` +- `github.com/multiformats/go-multiaddr` to `v0.9.0` +- `go.uber.org/atomic` to `v1.11.0` + ### Updating from v0.36.0 ## [v0.36.0] - 2023-04-12 - Furtwängler diff --git a/go.mod b/go.mod index 0193937f3..2e9286a05 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 @@ -16,8 +16,8 @@ require ( github.com/klauspost/compress v1.16.5 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.8.0 - github.com/nats-io/nats.go v1.22.1 + github.com/multiformats/go-multiaddr v0.9.0 + github.com/nats-io/nats.go v1.25.0 github.com/nspcc-dev/neo-go v0.100.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 @@ -25,18 +25,18 @@ require ( github.com/prometheus/client_golang v1.15.0 github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/atomic v1.10.0 + go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 - golang.org/x/sync v0.1.0 - golang.org/x/term v0.5.0 + golang.org/x/sync v0.2.0 + golang.org/x/term v0.8.0 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 @@ -63,9 +63,9 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -73,11 +73,11 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect @@ -101,11 +101,11 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 68526591d7b90d9bb38251f3bbd49944546896d0..29fef2509c848412ace06c3f171feb15d77f35e7 100644 GIT binary patch delta 1763 zcmZvbORMAN9mY`(6NYJr45gh(n?MeeEQS#|y59y0l`YwlE$d=k?Nu$wmTg^a$-0iXwP_qgEcN&{cMyg6u&hSC^ki`lGemL)-S$E~Ex-$+4)rwj_m0aC2@| ziJL=t`}g2==7>Q-dJ(5D8tpWp;^q$Sk=>%%_9}nVEHl}wt$DTEA|ooI7P3i6!)h8H z(&ex2!uYl4%yZNASgs;0g&Eh}fDP9zracdtt*RNz)`geQlW^)JW!PS&ueM)g_W}IC zZipYD%uQ)*iCc&yh#^to%YiTtM?;&HREOVki#?PF!M^H3vbS+Lt(AWBlsRH2sbi6( zBp2c5hB9&KzP6Acs>;M^<>4eBZOi>~lB5GAHVpVkKW^WC?=Fho7_V}F(zT>A%mX_b z*^DwBPX_#q$)>!~Ta}475Mm)oe3}YwQRaxAn7h5s)WM@+mxa)%QF=O?4h2Hj$y9CD zMY114j?v@PP~L^t3Wu-WYtInk#=Rh*s;l)X6~0~#X3oqW(VHTac6hY! zPs@qWe)Rrh=183wbLjCz;&M+}^a#!bTK8&tFmVt_E4(VjzmqB~={=;YC zyvff?_yWIp@o06~8PGn-82e=HX%SP95Tp^cv$K}XWTmG$IaZ5OlZ}eK#(!~wpM-GC zu=XX@*e&+aYU$46#a^T03bOh&skmki!mdo@a#@gvtKR`n-=S`bFPn{>ObTSG=}Xb~ z6(a0pYpmSC1O$RLSqw-&p!qIAq2kZ)1*>AYb1%F!hVpEIV33!(+u3=ETpZO-vh1XK zDoW! z1+qoL__iW<)C^)x-f$wphPhf_^X-TKc=aU&IoKcn5x|iDPS%YaXAtVOt}i-PC+bpU zR&VACD9PB2TS!_8VlYz=@MbNwgkKaWEdscYOkM&WogFuO`AY!kU;g1P@Wbcdz6|{S zHI%-o!FS?*Nz1coXKL8x)?Le~L<&{`idKVZoR$r8${LFmR+yK-4}gbfNAB|K4S;|C z%A0@$+{z?GU26%t!qhI@NUXayGTESs4Kxh)-WCnS)M1qEz)Yk3gExUk%u(g!7`2** z0t~Op*r6Gh9a?>TI4mT^mTMG5D1Oy2c##Z`=SeT&C%~iU|9%U&cXs;;(0o0J724y< oNjcv+3^nGYyp9^PmLjk(q{l<7#=a0m9sUIkFwcc2z@OFs0DTKO*8l(j delta 1761 zcmZwH&#U9s6$fyfVM-bjGA)_BvDQMWxMLKnL6;YO!tL0Ll>dYlt3ZNGP^=4RBueci~Iqf^F8Nq?|J{( z?SDVJ{lSAf42Dt=qV%IVzFg22ps_iy?s?!{k#Zzc%RGL<3bslxMl|sZZheS)o8@5G zG4zx{H;XKYc(MA)E8nYLy>s`quc~$Y=+2LIfL?-|q8gS{s2Zw)DC?kC;j$;iHJ!F z(p{KG`(x<+@I~}_W}!&?)nOgWeOm9MvcX`8Kaq?bEw=OkGv>bODrR56${|`k3E!wb zi(YBKV>>3k38_o;CfADnb;>z$A6c#0l@qqNrdt4Hs>2*GLDSbNN{ zX+D@yh>x!ef{M32+~DJ8Z#wBk%z>RkvdmI_G8Dh0R87yqP0sIfh?r`L8xb2X;M#^S zW#+yQ*0^WMYsXSt+Nwvgs?Xnl`}oCszqtkADSTEgV>XQpbR_4?a-MW32lx09GQ#GT zw=RNcx*!m7+AHnqgU9#jTAk%)gE+TCk0&#{#O66rS;gpys(SEjd>^z}&o4O3AB&{c zuAV%8{oVEAklx#&m)L3Jjw=z_fWWw&G13rd3}#{<72u|q^#SBFI$VU|*p%8(pl*M{_(Yr6WWP&ZwD@Geikkc_suppGRCA< z9ui@-jW=bhJJ3Q1XlXc_3591{JuNO1<4Aw_^i7DKMyF$jgsO)km=qs+c6&=18d2ym zSSBoGZ?Rs+!f=*ZOI@vg_m?+b0O)c0*Izaued6R&+&fJ(8>2CT=&rMjdA1mLOnuj! zg*e?D8(WsoBW)ju)hC~Po2uDa(=+I+I`EKnG4`22h_>mXuZu;ras0$5C3!2N_BL4d zCwleq(~INdr@y#;7XvsTa0szSi`Js$;7e%RMD_4f<9PSeKQthI>PypAx0DQuEaIpy zyEd|B(;$GNJIAnYT28zCU=vv*DQ;5Lv%h_duGv{%Se&!X_GG-{2{)6)^sJ`)^3%XWk!Eln(h$T7Yw%c#E`=BfB+^|)wCIf*;SRye8>=;QR*zwa! z`QLYr*Z)Ods(-?X`d>35`j5&;f^>m z16+fzHU9M`Y@N8I2bnqEjV6u;1796_DUFY3%Cdg3EF*7Cebrs4QA&}KdiAHTH(r0C zb$J!N-Qcf&^iJc|>kscYzTbevsnix)lq%2LHkz7pq|qbL9%!?UZ6JFBEE$!R5t5r1 zawj#ew(m49=-Qnv!IwH9m1gzHE$v{);{0^O*`c$`BiOVCLrdi%vbPZ;l3HK=^Fia{ z`sWWCFWvg`dJ?1pxXD58XSwKi@bPF4h*hu1J)aUB9JCCQ>SD1gc`fR>Zcx{seYf#@ F<^Qu*KtTWi -- 2.45.2 From 6f47c75e434143d39058ceeae2a1f028e6633c0a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 12 May 2023 10:41:04 +0300 Subject: [PATCH 032/233] [#125] ir: Set extra wallets on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-ir/config.go | 5 +++ internal/logs/logs.go | 1 + pkg/innerring/initialization.go | 42 ++++++++----------- pkg/innerring/innerring.go | 18 ++++++-- .../processors/alphabet/process_emit.go | 18 ++++---- .../processors/alphabet/processor.go | 12 +++++- 7 files changed, 61 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c05d065fe..97feb2245 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Changelog for FrostFS Node - Change log level on SIGHUP for ir (#125) - Reload pprof and metrics on SIGHUP for ir (#125) - Support copies number parameter in `frostfs-cli object put` (#351) +- Set extra wallets on SIGHUP for ir (#125) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 2e2aa1613..54c7d18e3 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -62,6 +62,11 @@ func watchForSignal(cancel func()) { } pprofCmp.reload() metricsCmp.reload() + log.Info(logs.FrostFSIRReloadExtraWallets) + err = innerRing.SetExtraWallets(cfg) + if err != nil { + log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + } log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5cd1b8dba..742f6a8f7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -414,6 +414,7 @@ const ( FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go + FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 1dc1d40ea..89269d50d 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -204,7 +204,7 @@ func (s *Server) createIRFetcher() irFetcher { return irf } -func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morphClients *serverMorphClients) { +func (s *Server) initTimers(cfg *viper.Viper, morphClients *serverMorphClients) { s.epochTimer = newEpochTimer(&epochTimerArgs{ l: s.log, alphabetState: s, @@ -219,21 +219,21 @@ func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morp // initialize emission timer emissionTimer := newEmissionTimer(&emitTimerArgs{ - ap: processors.AlphabetProcessor, + ap: s.alphabetProcessor, emitDuration: cfg.GetUint32("timers.emit"), }) s.addBlockTimer(emissionTimer) } -func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, error) { +func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) if err != nil { - return nil, err + return err } // create alphabet processor - alphabetProcessor, err := alphabet.New(&alphabet.Params{ + s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, PoolSize: cfg.GetInt("workers.alphabet"), @@ -244,15 +244,15 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, e StorageEmission: cfg.GetUint64("emit.storage.amount"), }) if err != nil { - return nil, err + return err } - err = bindMorphProcessor(alphabetProcessor, s) + err = bindMorphProcessor(s.alphabetProcessor, s) if err != nil { - return nil, err + return err } - return alphabetProcessor, nil + return nil } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, @@ -425,13 +425,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return result, nil } -type serverProcessors struct { - AlphabetProcessor *alphabet.Processor -} - -func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) (*serverProcessors, error) { - result := &serverProcessors{} - +func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) error { irf := s.createIRFetcher() s.statusIndex = newInnerRingIndexer( @@ -443,35 +437,35 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien alphaSync, err := s.createAlphaSync(cfg, morphClients.FrostFSClient, irf) if err != nil { - return nil, err + return err } err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync) if err != nil { - return nil, err + return err } err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } err = s.initBalanceProcessor(cfg, morphClients.FrostFSClient) if err != nil { - return nil, err + return err } err = s.initFrostFSMainnetProcessor(cfg, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } - result.AlphabetProcessor, err = s.initAlphabetProcessor(cfg) + err = s.initAlphabetProcessor(cfg) if err != nil { - return nil, err + return err } - return result, nil + return nil } func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- error) (*chainParams, error) { diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 9119ff201..8c8c13dc3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -73,7 +74,8 @@ type ( withoutMainNet bool // runtime processors - netmapProcessor *netmap.Processor + netmapProcessor *netmap.Processor + alphabetProcessor *alphabet.Processor workers []func(context.Context) @@ -383,13 +385,12 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - var processors *serverProcessors - processors, err = server.initProcessors(cfg, morphClients) + err = server.initProcessors(cfg, morphClients) if err != nil { return nil, err } - server.initTimers(cfg, processors, morphClients) + server.initTimers(cfg, morphClients) err = server.initGRPCServer(cfg) if err != nil { @@ -589,3 +590,12 @@ func (s *Server) newEpochTickHandlers() []newEpochHandler { return newEpochHandlers } + +func (s *Server) SetExtraWallets(cfg *viper.Viper) error { + parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) + if err != nil { + return err + } + s.alphabetProcessor.SetParsedWallets(parsedWallets) + return nil +} diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index b8d65dbc5..7a268ac52 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -52,7 +53,10 @@ func (ap *Processor) processEmit() { nmNodes := networkMap.Nodes() nmLen := len(nmNodes) - extraLen := len(ap.parsedWallets) + ap.pwLock.RLock() + pw := ap.parsedWallets + ap.pwLock.RUnlock() + extraLen := len(pw) ap.log.Debug(logs.AlphabetGasEmission, zap.Int("network_map", nmLen), @@ -66,7 +70,7 @@ func (ap *Processor) processEmit() { ap.transferGasToNetmapNodes(nmNodes, gasPerNode) - ap.transferGasToExtraNodes(extraLen, gasPerNode) + ap.transferGasToExtraNodes(pw, gasPerNode) } func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { @@ -92,12 +96,12 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN } } -func (ap *Processor) transferGasToExtraNodes(extraLen int, gasPerNode fixedn.Fixed8) { - if extraLen != 0 { - err := ap.morphClient.BatchTransferGas(ap.parsedWallets, gasPerNode) +func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixedn.Fixed8) { + if len(pw) > 0 { + err := ap.morphClient.BatchTransferGas(pw, gasPerNode) if err != nil { - receiversLog := make([]string, extraLen) - for i, addr := range ap.parsedWallets { + receiversLog := make([]string, len(pw)) + for i, addr := range pw { receiversLog[i] = addr.StringLE() } ap.log.Warn(logs.AlphabetCantTransferGasToWallet, diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index c2d7c1164..cd9088e03 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -3,6 +3,7 @@ package alphabet import ( "errors" "fmt" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -45,7 +46,9 @@ type ( // Processor of events produced for alphabet contracts in the sidechain. Processor struct { - parsedWallets []util.Uint160 + parsedWallets []util.Uint160 + // protects parsedWallets from concurrent change + pwLock *sync.RWMutex log *logger.Logger pool *ants.Pool alphabetContracts Contracts @@ -88,6 +91,7 @@ func New(p *Params) (*Processor, error) { return &Processor{ parsedWallets: p.ParsedWallets, + pwLock: new(sync.RWMutex), log: p.Log, pool: pool, alphabetContracts: p.AlphabetContracts, @@ -98,6 +102,12 @@ func New(p *Params) (*Processor, error) { }, nil } +func (ap *Processor) SetParsedWallets(parsedWallets []util.Uint160) { + ap.pwLock.Lock() + ap.parsedWallets = parsedWallets + ap.pwLock.Unlock() +} + // ListenerNotificationParsers for the 'event.Listener' event producer. func (ap *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { return nil -- 2.45.2 From 13a7a90101d2b84917e7f281aae1de6804aa941a Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 17 May 2023 14:11:46 +0300 Subject: [PATCH 033/233] [#355] Increase tree svc client cache size to test hypotheses Signed-off-by: Alejandro Lopez --- pkg/services/tree/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 56b97e687..3288083cc 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -27,7 +27,7 @@ type cacheItem struct { } const ( - defaultClientCacheSize = 10 + defaultClientCacheSize = 32 defaultClientConnectTimeout = time.Second * 2 defaultReconnectInterval = time.Second * 15 ) -- 2.45.2 From df9e099fa7af00e0cce242cf768b68ba5dcfb9e4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 16 May 2023 11:22:20 +0300 Subject: [PATCH 034/233] [#352] Add explicit max number of alphabet nodes Signed-off-by: Alejandro Lopez --- cmd/frostfs-adm/docs/deploy.md | 1 + .../internal/modules/morph/generate.go | 3 ++ .../internal/modules/morph/initialize.go | 13 +++++++ .../internal/modules/morph/initialize_test.go | 34 +++++++++++++------ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index aead65fe0..974c2a93c 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -18,6 +18,7 @@ To start a network, you need a set of consensus nodes, the same number of Alphabet nodes and any number of Storage nodes. While the number of Storage nodes can be scaled almost infinitely, the number of consensus and Alphabet nodes can't be changed so easily right now. Consider this before going any further. +Note also that there is an upper limit on the number of alphabet nodes (currently 22). It is easier to use`frostfs-adm` with a predefined configuration. First, create a network configuration file. In this example, there is going to be only one diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index dd327a4b1..ccdc4519f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -39,6 +39,9 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { if size == 0 { return errors.New("size must be > 0") } + if size > maxAlphabetNodes { + return ErrTooManyAlphabetNodes + } v := viper.GetViper() walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 494ad5296..9eb867faa 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -23,6 +23,13 @@ import ( "github.com/spf13/viper" ) +const ( + // maxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size + // of the invocation script. + // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 + maxAlphabetNodes = 22 +) + type cache struct { nnsCs *state.Contract groupKey *keys.PublicKey @@ -45,6 +52,8 @@ type initializeContext struct { ContractPath string } +var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", maxAlphabetNodes) + func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { initCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -111,6 +120,10 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } + if len(wallets) > maxAlphabetNodes { + return nil, ErrTooManyAlphabetNodes + } + needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" var w *wallet.Wallet diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index e2e5aa0ad..07d2da8cc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -2,6 +2,7 @@ package morph import ( "encoding/hex" + "fmt" "os" "path/filepath" "strconv" @@ -40,8 +41,11 @@ func TestInitialize(t *testing.T) { t.Run("16 nodes", func(t *testing.T) { testInitialize(t, 16) }) - t.Run("22 nodes", func(t *testing.T) { - testInitialize(t, 22) + t.Run("max nodes", func(t *testing.T) { + testInitialize(t, maxAlphabetNodes) + }) + t.Run("too many nodes", func(t *testing.T) { + require.ErrorIs(t, generateTestData(t, t.TempDir(), maxAlphabetNodes+1), ErrTooManyAlphabetNodes) }) } @@ -49,7 +53,7 @@ func testInitialize(t *testing.T, committeeSize int) { testdataDir := t.TempDir() v := viper.GetViper() - generateTestData(t, testdataDir, committeeSize) + require.NoError(t, generateTestData(t, testdataDir, committeeSize)) v.Set(protoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. @@ -80,25 +84,33 @@ func testInitialize(t *testing.T, committeeSize int) { }) } -func generateTestData(t *testing.T, dir string, size int) { +func generateTestData(t *testing.T, dir string, size int) error { v := viper.GetViper() v.Set(alphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) - require.NoError(t, generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr)) + if err := generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr); err != nil { + return err + } setTestCredentials(v, size) - require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + if err := generateAlphabetCreds(generateAlphabetCmd, nil); err != nil { + return err + } var pubs []string for i := 0; i < size; i++ { p := filepath.Join(dir, innerring.GlagoliticLetter(i).String()+".json") w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") + if err != nil { + return fmt.Errorf("wallet doesn't exist: %w", err) + } for _, acc := range w.Accounts { if acc.Label == singleAccountName { pub, ok := vm.ParseSignatureContract(acc.Contract.Script) - require.True(t, ok) + if !ok { + return fmt.Errorf("could not parse signature script for %s", acc.Address) + } pubs = append(pubs, hex.EncodeToString(pub)) continue } @@ -113,10 +125,12 @@ func generateTestData(t *testing.T, dir string, size int) { cfg.ProtocolConfiguration.P2PSigExtensions = true cfg.ProtocolConfiguration.VerifyTransactions = true data, err := yaml.Marshal(cfg) - require.NoError(t, err) + if err != nil { + return err + } protoPath := filepath.Join(dir, protoFileName) - require.NoError(t, os.WriteFile(protoPath, data, os.ModePerm)) + return os.WriteFile(protoPath, data, os.ModePerm) } func setTestCredentials(v *viper.Viper, size int) { -- 2.45.2 From aabcd8d3e42ad7bd8338f48d1e3c8ef0447c344f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 18 May 2023 09:28:10 +0300 Subject: [PATCH 035/233] [#362] node: Cancel ctx in the same routine as for signal watcher Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 2 ++ cmd/frostfs-node/main.go | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 987d27fcd..c45d44fd2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -321,6 +321,7 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s // helpers and fields. type internals struct { done chan struct{} + ctxCancel func() internalErr chan error // channel for internal application errors at runtime appCfg *config.Config @@ -1078,6 +1079,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) + c.ctxCancel() c.done <- struct{}{} for i := range c.closers { c.closers[len(c.closers)-1-i].fn() diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index a2024706b..425cf25a0 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -57,7 +57,8 @@ func main() { c := initCfg(appCfg) - ctx, cancel := context.WithCancel(context.Background()) + var ctx context.Context + ctx, c.ctxCancel = context.WithCancel(context.Background()) initApp(ctx, c) @@ -67,7 +68,7 @@ func main() { c.setHealthStatus(control.HealthStatus_READY) - wait(c, cancel) + wait(c) } func initAndLog(c *cfg, name string, initializer func(*cfg)) { @@ -140,14 +141,12 @@ func bootUp(ctx context.Context, c *cfg) { startWorkers(ctx, c) } -func wait(c *cfg, cancel func()) { +func wait(c *cfg) { c.log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) <-c.done // graceful shutdown - cancel() - c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() -- 2.45.2 From 0fc494637f83ec51382c2faf460ef44147bd8859 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 12:21:12 +0300 Subject: [PATCH 036/233] [#367] node: Disable unit tests cache Signed-off-by: Dmitrii Stepanov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 29625e657..cf2095ec0 100755 --- a/Makefile +++ b/Makefile @@ -126,7 +126,7 @@ imports: # Run Unit Test with go test test: @echo "⇒ Running go test" - @go test ./... + @go test ./... -count=1 pre-commit-run: @pre-commit run -a --hook-stage manual -- 2.45.2 From 100b1b5128f83eff490406e44f8af2b5905e2281 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 18:40:54 +0300 Subject: [PATCH 037/233] [#329] node: Add async evacuate proto methods Signed-off-by: Dmitrii Stepanov --- pkg/services/control/server/evacuate_async.go | 20 ++++ pkg/services/control/service.pb.go | Bin 100123 -> 146242 bytes pkg/services/control/service.proto | 103 ++++++++++++++++++ pkg/services/control/service_frostfs.pb.go | Bin 47352 -> 65587 bytes pkg/services/control/service_grpc.pb.go | Bin 16653 -> 23469 bytes pkg/services/control/types.pb.go | Bin 29301 -> 29301 bytes 6 files changed, 123 insertions(+) create mode 100644 pkg/services/control/server/evacuate_async.go diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go new file mode 100644 index 000000000..94ddc73d1 --- /dev/null +++ b/pkg/services/control/server/evacuate_async.go @@ -0,0 +1,20 @@ +package control + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" +) + +func (s *Server) StartShardEvacuation(context.Context, *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { + return nil, fmt.Errorf("not implemented") +} + +func (s *Server) GetShardEvacuationStatus(context.Context, *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { + return nil, fmt.Errorf("not implemented") +} + +func (s *Server) StopShardEvacuation(context.Context, *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index a126ce16d0c5a115340e39b37b8ab6427d7e989a..b1bebb1e27bb465a1bb508cb59b5814ec4a87ca5 100644 GIT binary patch literal 146242 zcmeHwZFAg4lICafS77vEPV~t%-$dEE&_!rP$yUTjc9_z{#wi?}W)mcP)a>Rq&=R!{ z{rAhvrvPLDRY0K$Ky-5_V!aelmGAjvRsn_n;fLtO{49$~XX(d&2Hn?L*zeTo;!WIjut@;^4L!O^GP?Y+b89d#!%|NFGR zy|=sl6W>YZ%f&c;8eM<9_#ufGpC;ot`C&YtrHlDAyRa>EH#e_GPNpM@?16`SJJI**?wiTX+}YaP zC^24?)wl&)F|5e*W^`jXk|29KTdX=Ue)(xMUXuJ#l8(}462-IS)%IrkO*V^9{$qajO9Q^pLN{PLgAU%Y?*?&ZnxFVBBJIyrv# z{N4NGllO0rUcP$$_T^DK5uQamn;T(#e}Dh>?d!KcX8~XN|6x>&>d&IxP`)pIee>Ht zUcMvE-2ZFIcQ11yIW77$nl9t$YIL1M6nU*ox4rpkwBS5#!~bM9x{4{}iQ@r(bZiUS4ip`xhE=6rkyVda~|Hb^2%29lKlE`EBdaDTOv~RI(qsnn#G?VwwXvk z^xZZ2^M*k)T`p$P_2$=AB|nx>Alcyr7Ahmk6-yB5KM5n3R~Y&0`NNy-qc|BaCfDhF z@o?)Az|x?$jC{MiI*k_(Z??7yxmqb0X$DpL#a$)t`D7ZOQ1v^>s^5tzZ`p+tf!Gtq z&3A;F$%makEJaq8m5Gu4gu1o@Usv+|eH8%~B=zY?z-Y64O=}FRfbhd4htX)WO2o{bfgFpQ6$LW_eeiVIRVripmZ;9;y}rIF z%?a1}Xnr1DI%JNnz?k|U8TMUOdg&9Qe~A>-<2`-A_Dvq`KAK;rkGP0DJ0%8>58avf)XN8c5HRygeyobt&fcExqgQ^B7@ zudRqWp*W?OA-{ZgL(o~({ZA@jlXJ_w$>=)<@TN-i>-;mHY4 zE*$)y+iz4L5ULipew$M~8~o&n+s6d7u^IlFy48nQ6RH6cpjo+cY8{5{xg6XRFuW_& zc9Q*#3xk=(Hf=J#jQ<^LiZW0dGHl=5n777rfB$=Q6a9^<>|_cruIdQlLB9CMd~~K= zu@yZlX#1VT{ZUp}uM%mOOqso!WRZ~D-T}GQfZf(+twC9*3x@zwAZTrXQUzcvqM&d{ z2{fX>Aj4Z8G={s`=DFIGU`O79cbaPimScCbNEq3! z>Z*iVPOev(fMN}G0)u)i6uW;6T_`fWyRH-&-QZGTP3W&yu!Zket$<-yD;AkkhNu?1 zHSJ6$hp$}Ve#iAfG0oAxvx0FPr?l)(X-6j8&a+n*bpNf~#?IBGrf7A!ZeTMmaMZR~2b+_l_(~60i8p1xB|5mF=a!*hZGeO93SY zR_L+D?0=IQ1GV}oG8yewRmPgsUzuTR-@h^gTEN>bIRlOp0GTUS54h_VOAywcENCop=w3-_U+3= zm1a+?k?bwBh)P5Tw}I6HmFtcyOHeM1Nae>QmNj)H|En)T9Evqm3~1kD$#^_?$$(1T z*NlvAaM7?P^j9_5!uP9cz_6>8jm#-SEHob1)QwCIU*W+0jw^>^+Uw$CI-z&JN7HGP z@SDyFzvaY$Je!}<>qRNQYWEKsA+JFvf1PDUMdtNbWmMFvmW;}f9t+0d5Cx;6P$%WW zLA9<}_=fjaD%iGbUn(kQ4K5UwnZqnE4(&4GkU|xS%H7+ShziM`))oaHT3VnKIjR9} z{1uiFg+AG;Y5fX9)-jad$*t<%V@Y^2bV<-kn#Oh1vc-!stT~{ zN@XE)$q>tjC;qySiQy>>xZQ1KP%L|0JMaVLD_SUIiw0Rj6pzAX(eQ@u->r26OD&Q@ z_H65{yz@j)iOUMCN6m03R$WFo2K87qejK7|I8^FnDdD7BS2z3<`YRl4;k7Ru4#Ngl z4#(VKmJ~nQrNb$QsvVB|w=W(JqCG7vG^g^?Lf^~U^gFV+NXj(>CARW-+oi%fR*kH~ zD6chK`uA8jej2`RWSV!d-f$VKe#KEUqrcL@M!sjI15RGCc4QGU#Jc0By5f-u;;kMS z&~^DxY`b559MeZ@E*J9|9piqti1YW1vPFk|_in8_7IeU!&KsMxhj-cGRAEhOh*LMm z;=?(v$2zjRGe{wEs@2UJ#6!Av8B#Z=zj7kBUjKUHG;VlBaZVp*L9$yqU|xb;1FEt< z$PSsihjdU_JZ7MZHbiMbCG*u5+`S_UlyY$~!c{i4$_B-_H7zetE}*_(V4nqMkR#}o zG|UXp#bU)BISuQT+DSdun0s*^-Q(`ni%b?SL#$kOO9#_E%G8$`2;4)R$t=-Jp~=2D zDUTPjS9eX7vF_c0^$S;cd~EAGmuMXoj=Y%b@5bcc_%eS~-uUh0t5z>HzR2iz@%<7{wWIaLonTOLgeHMp74v9Op>Z3YjW^GX( ztSLR#hkJ28-0A353q%?T>9ZINa?ZP=Q747KLA9<3_=fjb0Pe*p?}}Z6!M`$d zprt|Skhen&wJxaKy*=PpNcOZMDEO3@1I3*?wic)gHW=Rbv9;vltU(_kRC zW)0N4mrhWZL{NRZb?5fTZISD#Uk8tJuby#6z4(^z3#mPOTc|;ByQlu#*2It8Fn^!W zJ;>STvbr7gBHRNvfSuQfHqds;iVIF?rut}%CP|G!SeZ%})B zT65HC8Cre%UylrPI@+18S&!Di>ZIP*b!(M?nLX}1-H&6_HQv8v3CcofkQ!7zM_r>< zKoNrIeN~~%FukmQm?(b;*1Re;na?PT(q&Rw1NnyCt#wd?l;5Hr(aNZw^-zPZ)vDAm zp~rR6-XN!~8}#dBZPdu4t~`ZJ?yo|L8`l0Hc7v6Ji&O)hVV<7sm5ydN3Pl65GujPf zZ(pq%81%Gm@^LM%oqXf&*!rnS%x_e)1}|D!lT$uy(PW=K%hn)gxSJI0xo%~4U#)Oi zvwN&u_u_bWQzUdyx-x$avBugf9r$ii&st|S4eqmgWkzBdCKoeW4K3pCey}{9UzZkK zrKQ$!b;R9TW%&ghzF76Mw(@IOT~_(W_F0++IWO*4t&^fuN4~BcHO}j?5Z#Ltoc@hUG<)W5^v_AJs@We6{ImRaB#6V7)=h z(r3XL1F0`Y14t#`Dl()Nyuwb7MLCIGcZ-y_&>R`mMua$a{T# z^Xum3`EoXn9^ORXd4#J1v1eePAUw#~Z;xJ`wESM$b^U(*q#l9yUYrK^ST`7SJ@bcY z^LHLactY~S4-o{(Q3nVou!Em9wFJf8Zwxz|8zi=Q+-`n`HK9X*1aR_mtkGl^y;{tZ z^wn`Ri__2Z#mDGsbiKW~@itEB4>QyHhuRPMG+In%=_B<4CL%)9P8eL5$)kTK^Vu_8 z0;NX+kNc0}+4Abycs@%P^Xayx{vhHxPaoKH=2w$6zPe7oJouL=Wc&aw{7X@mPNVDV z>0~@g=|uxl1Y?aN)<3F>sIPfkN`bKy!KE5SH1$jon~qk*TM<;?G?WB2vd@|@$l-7p z&s1!sSocN2A|6~7sleawBTs|0rgx*6mm>5w{%qGB zqlCAllFF~SREesSBzp7w_1kwZ-#&l);^m|K2h!18B+=)~n0_;TLDiTlY&;vy#&I-B zgfacH`uuZz#-G>>}BYUeBWOD3PB~qigrNvberSoVyACIP%Agn{Y9le_ine+LAgsY5t6fH@H zf63hwrN+Oa^Xce$~Wb)|Mm- z4T;D=(P;K1nv1_7G+E~rTlJ0d{^@)^b=XJsEEQ9u0h7y$k}+V`p;TvSi|H<nodDy>VcNW6n5Sly;? z!3^CRXrOoY1bHLXBuyC`NiIRhx<0BrM@x1-nkF&#OrEX;O_N&aR|HgT6%~~`u*X;@ zVE|YwefmV}K|XI#Z;CFS`ILK9Wxa-nKFu?G1nhh9A%r^bua4N3hlf~g?q`rq5S6{F z#m_qo+R*!B=E=v=Vt)PW>A&M~n%tqq(my6i%DwSiB#TC8XAA0oBx!5$M=VbwFRPedK$`v4xjI_Z7AwCnob`bj-P=)L$RfXBMQ z=;@h1Os{*80cYM_dc~nWSAh#Pl{viBHI{0M?kQ;_&vca8`ybLj*5kD*D|e%fv}~oM zYrr}+vQMxZ`yV+d za7Q{tz1QfLtqMEtaXo_0{vaof-D-8xVEIVb0Z#3l{@_Mzz4ph6-Np?D8~5~KnyW(# zYJha^Hb+aT5b@h}IkMDbsd!l`&(Y>*@&8Jqgg;f0@Z%}@gL=U4l*h*WRGWQK3*d5yQdwrBatDs;8sV8(jUs%`e z+sP~%jXgX07yaX!-_+jE9$x2TG{VYLezFC@`Y6rb4yP9;=v@ZdYfNI^MPyq=$Lu5d zOUYs0J5bNnuV}2{Yrnae&(Fk>k!4X{(d!OTLdSe3=U?arjr`S)c(I^I|Ks_RUhr5@ z-I*_^XGTC4b%&&+t%Epy$>Uy_VV<`D_8-l!Q*3wA#;QH1WU}ggEwJvveQ{MSPri|T z)~8|4r+QMWlTO@cwZ>;gdEDu2@4rOdkCUk$j|bN%zgLGSQa@ITlwTfxuFnWa7#q6T znT>Z;sk|!IGGw$RW_bkHzGTRQHK%;m$?-4Ge?K~T^Xt(|&r6TBLR4UtBx-ztpfX8I zrr#i-f!FgHz5C{TK4%N2nwW356!K%f60ZsL)zCgsf0#p}76vHIdVqb4dfm#z8#w&~ z|NS@~>T~*v$Zt7z2Rny!Ol6PD$jp8kE#}}c%4kt{wv6p6@Y+l-5!;ipxa;bvO_Rmpf0gAkXy$RV>P!253KV|HO`Zr zIN&ESGYDG83}ZF3ouIckDl(|*e@7boawsSX18`oA?i1(-Ici$xtbUMQS+6D3S7-K! z_4ne~X`T012Ybs;gP?tW{rz@}U5=DQ{j- zUc{rbNH*yCiEy?vPkZ?Dh4kdF^BJ5-5}!c6+q1Z}+;j ziss&-eM0#lC-JJTp4Yy?@a?5A%xg2d)dP4n67{rAU(Znk zT*&UY9h>aJ zsK(M_E28FQpjXpnaw(rI{GV3;U)Da6+cH(Cb&u;4W(PU6=T@r|i1m@KYm#f{^a!o@ z;>4cYxWTaMo<2}-Ts^So6Q|q8Shi76LELX{Xfaf!`F7o@ev0Q2t{&Z#7Jb6oAjkB) z2I$~Taxd*rfK!I<|80$XaYoPMl9eFN^3EV_WAG8ZqxqOV*LsIKxwp%!Q+iuj&dl&8 zseB=hKlDc5XOLqw^Xqt#eS4Lj7LJ#TMLZjS30y+w<0}j4;*MTU^Q+#nl}j&7C-G=8 zzKr781-(Qf@|-R4C0Kf!0$;?S53QX=(|GiWuFR(N%{3Z0o?p{f!|7FP{shHQHX~$L z7hkP7$+?&h-r{mWsiaDEmR3k(de!+=IIoWB5y=k*Ir~^usgowvMYayVz2p09PQ|`! zf8Mcb*kIJIP93Nz%?~=}>wK3ec%veNywFvCj%<~+eiO4YW`%6jmgi`u;hVLY_NsK= zjXu~Rm>)Ei^nIe{ASWrEig(caI_ZYsrZR5tJ@3K2I6dibMQh}=yfR3yn;)BOdQU;# zkd>Dd%pNLzwD=BnzORqu>%Q{NuAJ>0_CVub|AmDn|5kCjX-1!*J7%Xshk4^FYICj_&3Bf?$?5{yk-*FDHLBrzIeAnj@Lm2D#7ed--J$|(o+U+P+ zpYKLP-6&=#Y%9(B#O+}Y2s%SbY1kuhTU_cI@c_@s^d6!6ejFTZ3Wi%^x8?9bz&rRP zVfF;BXxJ&}=$F6lM%%9UQbQ!IB3a0+KEZ5|b8jI^c7kVNjOrk*1%)0_?Oq&-3o&&t zu!RsCW+!(qKP1=qyx%XhLga7Pq{$hIck`5fn<;y`MGwK^PowdYf7w=EJ*D4hq9exi z^F~Yl^{9w`8F4Ycnb3FC>35!zV(?k?zoYZfWJ=#idiIRIJ;XF=_-vj=7t7INL?0-k zTcdOSPSJQyzmgl9A6Eo>@5yBKcB~LBJqg1#{q&0;=VdR@o#kplmCcRfj+di4s-0P) zd=|X8UgrD?CdjOru{D>yJk|MY7_p?Lxme~V%l?RVmBRLs4vG)jC+4FqD}5Nt7?JkY zA_|5v|7KMpHJio2FLDJ1{)rh}J@@js_!bo?8++K_&T7LX`@EJRSGJ@>rL$%J5c7<3&k?}C0FN{14JZc7IeTrqED2ilPmg7w$T;;@ai!xKTkvDeLtJr zyvqpaSJWZ&lR{|2@nmFv2EqLMdHF{W)VPWs^35`mmVYBVqDPe`n>d^h=cIxhYBIm0 z&G_j2sKSqhTyiYx#1KVL%OyjiWMg}@7BH;eQ{dgE;ZxK4?Dz&b02`DDJ-2+yp7k4$ zfx`B^l)V>cV}oqnLDy$Fe~2aRUVd6Of2JO!t87Y2mt&sh(-&Ck-r1L-dOuL!*v_X4cqoXbxLMrwVLIRfrJZ5CgYaqc^WRI-Tl;wniKCvR^8-~ zxB(wB(KECR_?!u?d{~#F7sRs7{Hj-w4Nq)ikVwmM%5arXaL4$@75$gm{&YT}Rou2| zusBlbJBtZ@4@?dY$;e{Vadv%kV>$acn}43Yn#5Cj^RpQ8USHqb-1sL}oSqL>sb!%o zO-l_PuHCA!vn{b=n`aa0v*{b;sQd~cG+Ya4+q#2>&`GezJ;8f%@P365>b6-ddv&nZ zG5fS)yH2hwF4<3zk?I9L=2us*WTdVW+40Y;=JF4~Coz56kbkZs{c;@#_AdGSZ&)CX zA-6Gu+G%n8>N|#(iiAh+q$Fa^D>C zCkAewU?iVjy*cE!I&Tj3UHKcc=sGhv6cUH@dJs+KFR>UB=K7egv}LmgYCFOJ5`i2T z;BWaP;?0ieVPp&tWf%4kOOQQ8gYFE%=CXxg23mtdZ4j7CtZaM(AHd3)5lSZ2HXy$; zgCB*^4nMIp4u^3L_)LXy?4jHRdqYC*9>BeNB(8{69)p324I%MFWHeV~D9C~3K12~K z@5tYW5>du1O5dhi!_9Z}(ygNUM22bD?7}@@Q!0cfLE2IxG2+SVk17+*R^kEIcvPT$%!-J&H6J&z|ZD00~mds!uL-F4; zDKfw!$OwSLb>OAI*<*=4c*2w*+(jH>F1m`mIFRCw7#C)!4ZAWek8v7bs=#bIJH%N7 z6KWw+BZr}rOI~7Na7;W(t&sSj-V^~uO+l@iI;5fn+L~l{RtPZ>>hSGsO$>~wg&Nuh zDp_J8v6pT}kOwufQMR>Bl(^2Ux6v9ZV;AfQ1yr9{V7q@q&H#BoNGaNiH4Yj$Zb!Tn{q+iZFF5 zNU>~;BVCh09kstS#7$sHZ_>Tz<@kgLYS3)NF}f^2N)%gV09JR zbmEnLtkHBHtPIPffk9>QT#m|OqTu%Z4w=X_0A?!z8Ytjh8JyYG@IJfGU-TOorVb0Qd`d&7=y^Pl7b;5Xbix4N5DhJB-KY1Mal9kjJFkm(iOAJ^tT3wjR0UxtCHJu0i5fG+17D$+O zVG7d2xlJexMEzSr0x@tRg5-fX%5-YMN!QRcX1L1?I)MaswM#hB0Li8Dc?C#dA1DfW z5oHEC_2nys*ZiLH@{+8FcW>(o>%^CpRTay%FfH31zWCDPvdzT~Q5=iix~tX_GO+ z06C0^u13_Vt25U79B3L)#8+6I5d*RZYd{ZW3p=(dq*a^g0(@3J%0ipX!MY*HZDa( z31f35lZ$<(imTbEQZ*6aVVrD`1i>mqEo>16214YncBVI15ZpnpJpfx^G@w>iZ&?py zkFM$h8_4GjD!MQQF>_i%oD7&1SxQL4(;%Zbe+3}9b)v!6Vj^ehWd@zaz6%A!L@)w8 z0e8vY#vn3n#ol5hdGG|=HtLa3jwB@?AXs;Wr3qFskzw!364n6w%*bIZeVnL5%<{!4 z{cZFU;TjlNq~Wg0-W6?gvo@7nZCHljL0xA~_L-;#Nl0q*&Vo+^Xe`+auj~SHvS5)Y zQf&)T|3F2Q0W~oW{x^|KxRN?fgPy&GUH598O|)pMdNiRQ6rXU zAl$;`GRrApZ!M=JpGr_KVG0wuTtt%Ad&mNI*Omk`z#xP8)c}RNRDhZXi%J8BdG$IO zDg#H`?n#skLK>H9U*QA5@+}QQ3}BBO_QgaM^;f`sBJJ$2gaOJS62yb74UmvooY<6F z9;HD)4J0$@1ozoQl?Wu*R~o3=i@1taZX28t5{MH7tMzP90#`!|-4zC?S=o+1$?_P? zGQgrMhz;m!k}IT-GFYT)Sz{wb%xcn#?}~7FRw)w&J2}uethaevvs2^bDaH)U3P-uO z!xUo%#>xGpn9?BF`fYb6$sX)vCx$>l{%^g0;=jj;LlupAXC+ajd|;47h?>?4t9TS8 zz@pOF?AD`clxXTaI0x;hi71GASyL&j6L!3=y)1uQrdo0d8T z4p8_;$Qeu%pXCjMFaSovzACGY9br19lQE-8u{~kq#56Gxh-x+}#a1v2nRD!d&x(cMNrH&a;{s z>||+8HtjsCnSpt%Q|&$hg3u3UL^#ObfmzKAeCw0d-t-!apn2^R6G=J5lVsywM7_w4 zD~K^j-MIMX5Qo{ZZ>llJyRr<gkBGs45U)+mk(M>K`%Z4p)R7H zb+YqD7(hWO_mAao9alAdXJHDo3a3aPP5tSG7pl1^~!;9^U?3MolBycd%GN2g5 zB$^V-yuEMEC%2J^8H+)l=w*n+p%cx-$r9s`%YrjFzqZCX3KZZ3z+Qb47+&bHkjg=g zG;?BFp@G%7w*@u7uEIcKqZ4Q&Jxa*N49u#kWyx%ay=*6g8FUD3>U|TKo?;@G zb6A^};NeV9VbF5Nt7bEw#j?%I3~H9qJV8YlCbwEeGXrdY8wGE-nNI@kph{=v6%zsf zRN+K`ntlSoDsDIBUX4U7)j+psp!$YeZswB)5CV(|BALNY2u-oI!AME>8jK`Dfq5<_ z64JVx(+`GBup*SjnDCYbi2lxl85pFL$+|Y&emS*=XE-Y(C0iVztwI@Basw77G7%yZ zZdqiAac)3nP`^pjh?hjJ1}>Ek#8Jivk|u$8$%gX7B{S%Je}Y&VAAGZ2N9Svi>^fwF z!b`L>KOw|*s>;X%95OK;muY!;F*y}{z|j?=OTrZSi4IxEH6#Vt$z3al!Cvm>LCFr6 zSxDXkeT2Iv)M#4xx&a^X$&*(!P&fnr8kVhf18N_13NtA)Fl$Otxdr5fnN&1@M71_u zIiE!X$y7ZC;lB(V1|!oH#eg$`(6hLf($p!!VTB${|dCmo7ULv=$TLRMKH zE%cF-q1KxSFd(1>JB|KgRX7=aLxZcebYX0mfiZ$eWY|Dzji!)P%L@h)AOK2BM9CzB zrdI2yWeg_@frO=TzO0%kg~dejk7}FcU9O4{rVc_9)kB2@ z_9r*l1RKGpP`>I3lgfDT%waPi%m_OTqOr(B|mfxXoW&#Uebmc80ml{VpHQ@m<|?K z1=W-qx!$Om_(0r?-@OSZjkPga8>6Kdxl~q}0b)GwWS|=T7q4q$GG6-xoIDmn$Pc`; ziI*jC1xac~;(;}I;`b%YAkhgH2byAxnJlix-ym)7iWvG&f8Q>9tC~(*Z=8|yMs7VEnV#!!p z*st(Jff?vEFzV`RMbv3DW`G;{i2;(jL!&7ZWmY2wRii;CUu$e&&SgesV4h)%i&lQ> zTIem=V=xL}=cr1m)kPkx>=iK)x>D38lpUB$RJOW2kYFyXb_wM zd43Xu!1OFkWJL%ZUhA8{jYDA|)^C-cY$BP}n4Wb$zz$0+?nT#RvKlP`@<3&eX@6|J znSsezC6ReH0EDR|6dD}EOmd>;C$+>xQlQPT7ZhuREJzCpU~1`&WDegD4NNi7qy@0B zri9|CclAP8S(25n!pVC5g?-+@mkkR&6bj@<69%`J7g*o|#6kmD86Br_pxD<60m%>g zAt06-K>^gn`VM9hHaU(J8u)tb#^Y3+6Lx6Q75J6d6dHg*E9qDdNR7;gPGYXy;mn}^ z%k^+LkY|O`&kSy9<$67oiTshkBt(4#f+LeF;D}C}ntNdY7fB}pLi*KzUeU?Lp=AhI-q zFllKpSO5;?Q^TbRee(-%$VRvkwzIQQJB~G z%kQpScH=FP*LX_|RYimh?6Ng2n?f1N_ia0qbgH}{F`b?h6QQoE2~Js9gO;L@ZG-`w z23O19UN=f%DY=mu`@P5vFrh=Kb8k`LG6UqWWkF~dj`z`fQ419Wo z=eGFL`xYQCZYv9aX~Nsh=?SR zFz`s!*5SJ%VkvNiK|iu{5c>*_C3~=wfvIpZw)c&RP-;wS9w%0IfYS<=O!Xt z?iDeSa&mp}t)P==5Q3&F=|nLp8o*s4K`Cr<_1Roa+A{-8SV4Va&A0C0u|Hh-2v*td0Zz5%+5lN-O)!eV8D78*Dt z6`exXKlJ;sMAUdo427rA6q$tvruHJOBr4!){0Mnt#C!#)JUa}%*X#4=Kg5As%sHPG8I_=y@^ zF%y8Ml8pKXZ1#jrzStuln4FieHrwdt?Hn0uZUZ(XS52si!$Hol2QXlk_8erFD>#iq z{hjgwtSS6K&IG9Js^IvAxh^D3EfZ$G2O4IjZ=}OC*f;AeS#c#V8cXazn)flK0_rh3 z-09n7n_o9KKaCdA!_AHJ$uvH>{&;bc#EVapah#lt=d*M%pJo?Mt{3xkezF*SK8lm^ zud{I+C0}OaZT_qE^rKNaik?L+nQm@vZk{h^|7#g1>DI&9 zWNL9Jcv*iDl=t9d*9gkCTdAJJf4&w2+;87*SDp*&;#W&;+J>(1V|1n7rzpbTb@t+j* zD}-2jeX*VJO#MeU^+I^^yyV`S`B~i3_(cAvg1m!}i62G3l>$V6SLL?NKxsf#?{YDp zP5wK6w}@LQ!5mZ)A3G_D7UetPiZBnA$rT=jA_o3qu~$1^FcX!|E=sIM>3X;ll4!ds zo1K)H#jEKuxqLAiU$%0DAS)}OlDR^uOumiyfQns}%$j`RgQNL4oiExMUxbH+R5DwX zSoG{dOR)_M?W#<+DKTxr|3$fm1}ec;*JFi`wc*0-G8!H5JQfb{l ziPiECX;h7wsr+uD1S8I`OASw;vl!C%RH7fJK1yCbjS6MCxTn(H$*1sDwfm8lEu`0B zg;dg4DM`!5+E};1JCqAEQK|2slyh-+LP3m}ivj>QB|oEZm6(x&g9;;1$~js6k0ei! zItF<56gX5VxtWBkVwmhJV6d2!RME{)J54xq_7x~pDCL^yJM9z^9tsPplyW9^Gt){H zRzyQUwSn_+X{8Kv(Ewm^DS4&R(}e58Y*hNYDHZy-+pv&uW=0AEDvU}hmHL}-g`^MI zS1{nBlq*!a*~UYxkOBcGrJRp$cG~Dd>B)Q)1XL(Fd9+akJPs5BXdWfMWUWt2B@E0Q zC;+f1npKITlcVKA zw$d`*9HN}e2MRR81js0PTgpb~=DZb)S|twLEri6$jp=G|D{=3UJc8a6UY zL^j_CYAY?-i7l+)Atyzf=KR1Uba(qKzJ7gn!#n1(3BA8Z+s}N)o5-tZvDOC&$y3@l zewxy5`^ju{6-VFy-+as2F_<@*H#Z*B<5Qxm;=(aB$%4*JTt(6zJ@O)QDjqCyCYS@| zv_jtqT8RgSrI}vEHJP+J)|OM{SaoO%Dd{mYTU4c(Cf8_C$6#Adm0OOnEd-_K11h+OnzaDUE9(9Lklo zQRv+FXfk5WE|1wq$eM1ILv@o{=zub&_XgdB8_iui%{IQuui9BH)IfRD+k*bxiY9k7 z&9s!!n}kghTj+zzcc3!TPrT7JjZRmn8z5q+HJ(G9EP7-;!&b{Ga?=kUF z05*_HZ!hsnd6lk~IOY;S(7S!KKLW|EZ94ZlUW9KUH4sqm$T=1)EwS_`ve*T~Xi=ft!VFb?1 z#@8=bndlX_Q$%Q-B)!YVBI0ueL=zwAF~5E;&c!Rsc|< zTV$4pY%w6GN_B&&9;z+Z5>=gD9=du+PI(u7@1fdsEmOAFfWe!jLnX6F5M_mmmC{r~O%KpqPfW$|*7*WvWFNd?p44QljU!(2_Atc(t;`dDrdn9yqi+o=sPktv*fw=p=|CJ`&^XXZ z@iMmZXQmHCd`kJoE1t%B{=9WyiI1IQXu*k(C$c5|IbFbtk3lg)(M6N?#Glayc|1PJ zpeI8JJ_k#3emMN`>5n^qM3W>+FX`Oa^7Q*D9b$;i=ZoknPA}(Yks?U#1=RcS`0(lB zkG0c^mdKj*_|QEozDeU*LI-}L{Ml;Dc|zeVE_&(@J5PPw&L{EoOmU*qc5-}eGMl7) zjx6{wvYGI;-RZZC&F04BJbD#&zAyUQvxp8P$r(2sEu&**bqD6yj)^%N2!_J$Z^hBD z_p@Yl9>1K8PN(r%tHX2I*{}~ge{8?Jxt>%1z2H-Wd<^eXqz-jGiYENeEqzTmZ6u#3 z>G(4G6g_;KSXKGL@_9wjk7mghgSV|9Q3v`u|~O5QfNcPqE)7>FD{ z$cOBehb(6wXY3Ty&hSoD0;U3t@Hoq}n%yN0`Br}YY`?z-;Mw$fUKQ+G{w$x7=LOx^X|di!gw zBB{HcTbpRT!l=8RJKOPGs}Sn0;m*vnUg6VWx9xn0u~xy?(uUU%)c|9RCq zh1XtpeYoFX>lI?%HRR4`lxr1T-L>UTTdh}Mb=RD$H&V=7G5A=)dOWM!>#jw&425V~ zD@%3PsJkE8t(CF5YuhWvTEo!lu8FTczFVuX>aKye47GkCwclOyUU`(aP7$@=UC*xV zw1xrIVXIy}*uGXF)nTJi!W36Fmb=R(!o?@+EV0G85 zmrS*WVbxu$t{-WwUrcq^scSo}UqBsn*QjR?xYjP14!Ud7GaId8Aa&TFJ0EVXUs!e6 zs5>pThC$X{!|s2?wSIxtUFYt%+8PF1cP)I7*-9TSUXKSn2i-OIVMffpZ?Pr^wGO&# z_??ek*AMKv>-?PtyX!bd$0p^oC^u1(E|%l;Pr5)Siuu%MaS-(#9aa60*Xb)#%#DX;rHpot#3Yo8`4To$$$l$R zM1-QDRPx`TWIF2SAv02;bh2Z!MCR+ZIQ=30mgcL#Gt$rawE8xWt~(iRen{2+-Bdch XEGK3V>*oIl`pN=4 delta 3747 zcmZ`+YgCnG7GC>3M5SADNQ9W%znS`h|Z7s1K)o3<$dGFL+_fHjyy~O;sAK7@CF6246h-p%0GiCHATCwSW8| z+xO~3qT*Y7Jl{F><1!WIyy1;4ujgX&{wQ=b%);qmA!rO$Vb>82x+_Dm&G9P7^kG}L zjMJlIAs+oAWW!nd=~W4fcjj?+AeJ8Tm5-kuCZ2<^ygn4S9@@fwZ}cne;R*uxao(%@ zTEd98^ROks7iYw&u<$b#)-=q);(A{!KT?TU)`xqOS%JfP<+$p}K_bXm^u=Qmmb@!6 z$KcD8g?<{WA0hU$uI}4iTd82FpYBaWmvhK6cHQNtUW~i{#wVQDk>Oe0(|7#zo z@l2@~ZmC_3#j^r2=d21-I&R>dLj&woZ2#T_XV$sJUg-KPAM59apksjw|5`Z*UtJi8 z?k)rQTOkT_=8csd_2+XrP+=DEln78JoFhLQaLUhl&X%s@5~dteIp6Bp1pLsC0Z)Ze zX$PpB{=M_P(Atpatp6?mu>3?H7G3oxM+>O1tzYGA9a!jv5si76v&bKFmiyqIp<&!{ zy3bj3RqusI+VXJCY6BhZ1Rva4IU8Lo0`cyT8XUPBi6frp=_i(Ba(f>xc+r4eZoPBp z`bG&4oE4ctxaO9!a;_BybgfpiZII_avcmA@AB4FICv}S4AZOF?3NQTd+kAZQPB3O4 znT_ui81UBJGK{&ns|%BH^x{d|b9xi{sa;$>aq}`P~ItT)sgDh$B=u`}0dw z;|)jaCT2qCd_Sut$!Oa6`&9AQWT!7bfkblELIBvQxdGy7LN!dL$LBx@xu(H*+FA|C z!XH8dGr&rP)!h zLB{-RxKQ?N7$@SleF_$uw4afqWe`dgE5Tu0e&KBw}X?O zaNOr(D&~NZY*?PtkZu5pxA%fVSM!3=u8 z)>D?;t61`)S}=$s%`d=gGVbT}yD!5O`o0#d0{+_?m^n&toLaBrv96@RdPt$x0(gWf z>UdaB?%{4L4l{{=7s8XWV30@6A2V8h7?MSu_5v2xQ3dh-HoLqVxk+-ZT>A1DYqNPH z55BWrIX#Z%Z&C)&gM6F{7c+kzWPpI@P|e#4;oP@)w5}=^G3_`@(&U(9k{14Qcc1h3Q#0p4a1jFjF0j|lR= zYv35yGFEV31NaN%3r5POf`Ou@K@hp~cz|Nd>yl2uvl5zuwA6bNVkB!S)tqECw`52f znsy3;vAuE(oj=7nC8v0uFLv?M#75p^RU^|cy$M=+y%8psf`84Og)5E!+2=gzeIXC+fbMR?mb5MB0FGC=$7UtY0R;S`Jdrt|os)@Y= zy&%6*DXy6zO*2H64fD|@&3x3VW>$J>2RHizI|UQ7nQ}S=y?!q8REr0+(CvP{!L@AD zn=SlQFG_^B!X&lGv1O8JfQ!7+%7b6j%IWQ`ti+*K9`wXEejS9g@%)MhLCt!it1_H+ zo&$q?Tskd015bF1D)i%3Ub1s%AWjsum}E_&slW{?n)$T{HOnYB`88N&9mXFFYWk=X zk_AI6Z-6Y}o-4d)!RHj3R;!H0(k2FI#vu3Aa*p3Z^Ez1Aib4L|)#7yCEA)6mt=t6Xv#y@TAr(N2fj*l7{YFgmj&! za@xa==XxNKo7vIx;N%QtnED||kSta*UxFo~bjc;imMk{9a0z~^zfY32;h9E|2-aKp zMI)FigO=x-DKgJrhAhdFN=d!28?S&x6s*1i3nZ(RZeM{#_a(9N<=`cXB<7t6Hu_*d z#H7--0eC{P+Ue0jmXYmUgD_WTO_Qq&Zlja+cQfU~SHY~aDotC{{@0tPzn{jW_>Cps zPLtlMEQvMa;ahdW4M{qiNj`-aZS-c1`w`5N%}kXyA&i#a^Sq3LNH|&DSACtB7h# z^OXDeSQP%8s#F`sNyESP=juAC%oAr%5GAkm^g2>2a}swqT?NuL*y31 zi0P*5SY;XWw7^V#XxlAF)Ujl;LvA6G+Bw*>Aa=W;`>bS5q67IZ#ZYNowm z93K%Y8EC}Fhq%HOvx|O-kg}*yBL$$TO-+7rTzhhq0{LLT9JTnlP`5Km3IgiZGXr;* zj}o;dnoBSwf2o%#d_`48#-ih;(Qor|#go(?!-N>WM8CxH)4T+!izZA^o(!}#Q946e z6FId$S<=!6Nlb9_B*m%*; diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 32a87c744..ca35ae043 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -29,6 +29,15 @@ service ControlService { // EvacuateShard moves all data from one shard to the others. rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); + // StartShardEvacuation starts moving all data from one shard to the others. + rpc StartShardEvacuation (StartShardEvacuationRequest) returns (StartShardEvacuationResponse); + + // GetShardEvacuationStatus returns evacuation status. + rpc GetShardEvacuationStatus (GetShardEvacuationStatusRequest) returns (GetShardEvacuationStatusResponse); + + // StopShardEvacuation stops moving all data from one shard to the others. + rpc StopShardEvacuation (StopShardEvacuationRequest) returns (StopShardEvacuationResponse); + // FlushCache moves all data from one shard to the others. rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse); @@ -298,3 +307,97 @@ message DoctorResponse { Body body = 1; Signature signature = 2; } + +// StartShardEvacuation request. +message StartShardEvacuationRequest { + // Request body structure. + message Body { + // IDs of the shards. + repeated bytes shard_ID = 1; + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } + + Body body = 1; + Signature signature = 2; +} + +// StartShardEvacuation response. +message StartShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus request. +message GetShardEvacuationStatusRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus response. +message GetShardEvacuationStatusResponse { + // Response body structure. + message Body { + // Evacuate status enum. + enum Status { + EVACUATE_SHARD_STATUS_UNDEFINED = 0; + RUNNING = 1; + COMPLETED = 2; + } + + // Unix timestamp value. + message UnixTimestamp { + int64 value = 1; + } + + // Duration in seconds. + message Duration { + int64 seconds = 1; + } + + // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + uint64 total = 1; + // Evacuated objects count. + uint64 evacuated = 2; + // Failed objects count. + uint64 failed = 3; + + // Shard IDs. + repeated bytes shard_ID = 4; + // Evacuation process status. + Status status = 5; + // Evacuation process duration. + Duration duration = 6; + // Evacuation process started at timestamp. + UnixTimestamp started_at = 7; + // Error message if evacuation failed. + string error_message = 8; + } + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation request. +message StopShardEvacuationRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation response. +message StopShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b9b865a90584cbb61b5a30056aae37d1918230ec..979c226859a0ebfcd3a5333af683f48efed362e7 100644 GIT binary patch delta 1743 zcmbW1&rj2E7{=3Oh!V5Ne&dG95Z#b0bKnMsC>WX9fiX8AF-llr-PkhLrR@wF2$M(% zhehH$JV>HTObnbD=|K}toVaNA2dE709^}Tw#I|g(z}rMU_I>(3{e0hk-{)Xo!Dp0( zgU{e!S=0tOlHr0ABopp?N{**VE=DI|%b*2&LC1%te-d(Uti$&!CL=nH($nqGk` zcjt1r0=qANpe>N{BynNy$+iYNK+&t??S+pr@I`MsymP%lE%||vQGt2#WY@9`H_^R= zdckihg;~M+(zf~mb!zOJhKmw4u62sch{R_a%nGCqnFby2o10e{Pk&1I>s*=@2X2rN zf%edt>Bi+R?J_NFfMQd0lBGuP(cx#KL&@0GU~H0NIdU=uy{$Ji-X!Ph2*#dM)|+it zrtHIHBF%5Pmx=!}%)o~l$q+(WlT!48eTNQyEzaBjj_-b&IlS!xn*Ib@kQ%3xVV2)$ z&Jq7=cMRspnQJ5?$gjS|hUvGM#65%EVHNlnl9q6u7ZiI=1dv1X;lxK@j6(`YU-*W56g{5koCFi z0_5H!4J-{^6Ch`dPAJqHOakOsI`Wu~L|BTuJyMMfv)NX{E%ux}B96zKE(ydZTl$Xq zE4KkVg0Dr-54dZwt>JwwD+;-#c->vX>`K-&Rvxi^)dV>@g`}&r<}#vF+;dEmG8GlX wNW}zk!O^-9sAyftQ#C?diBq(voTiv2g(+evWhvGGLX@I)AxRM*AxQ810Xo*FTL1t6 delta 9 Qcmdno!1CiG(}o{A02#Ffg8%>k diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 3fa1e54dea1a986d8b88cffead75e1f45ac1d718..96642153363125f6062c7ca65bf27631ad2f980b 100644 GIT binary patch literal 23469 zcmd5^ZExE+68;?j3SJKAIYlDx%ZJ0|P#hL%yBpy8f~LJMheM#qw5_Wwc_lg7T#^5N zGo)w^rJ>B$a_wTbsLA2XGs6)%AWcBnT+N|7U%P*DCZMwhwcPauYCQ00D6I{s^fa?C)3@hqq7m}@^O(?0{)8u z;c}H`@}f?cQp6P~R)fJrT?m<1>r#q(5!dh+5L@fdU<9&(MjRj4>0KtpLs~C@5~d0D zF(ycBB)gMoJ~zUqBA&--Ue#)jKFHtKX$cz6KfJ%X__a_|@>BGKC`ysRWS7whOjXt5 zR=iNGqnq^#t@&OX`MUa-p5?!e8bYx0m7bRyJx{m4$vCSQR|}bZzJ0yUvNy6`6w|jc z==u;y}SFbOzO(ZAC>8m zL0EM8|1+)Xn?+ns``P#Lw=BBsK|jU)4ftyx{vP<01tZ`wPZnj7r~i{5OX-)%$TMOV_hMQk zf7a*$W*)tj4>w9cG^!t0vW4BK3Hx9_wcJImz7wk#ah55CZ9zyQ(H;} zcE=4hht(q*osh+Nj9MmoQ&np=Y_8`b&Bb#05S(BP8NkZuBoJ{xkP_b&>sr7hic0NU=q#~u5xW=pVOyFI)RdY5t^8vN z{<-m41f}PHv-A0}!0-)K2pX53!(%rySP{TTSuqFy#{e3PSE)q>*O_W1lXRAXH^3u{ z57&(yh#P4SWS1}qb$bW{+}R-oI;zG$jq6x@Y>@+Ov@W_c>QKO6Wd+!sv3}$ZSqSdd z00uR;>*8`z!m0r};8Kf|5TXRf%C8GY9ikFHj?v7w&BH39A5*ELB77u4VK{}rMEC~z z_3(`d(ZxTO(T9m7aJ^f7!#s7O8ozps)DX?sVT@8XZ28J3#t!sjw=^M;DK&#sM?eA= zBE8*%>DAT_N5HyVC*V@}tdNBY+)C#2G{?0`5B4L~{UIm~WeefcNQW%A;lRPZIY2fc ziw21CN|NZKs116qO*k_G7PEDp2#A6p8tQNv_l!N`LnRQXpEdD}!}TReo;Fg)iFnQt zoe2^T`3*>jLagYJyQV}l;r@L20^*^EF+&M)6~tFakVHumUFY}3XSLm&x6IIUe#cM` z!n$=6+d*gtT+K^jnk=g;>)e@3kaD8E!=~_>dN~k*VK+mL|l0l)Be^UBYnd*|CQ- z9y|$~z}s&=N)=*2m%i789BH@_>F9%+(T(hTEa?&k8M8g4-<~}6SD4gga7e2<=MKM3 z9kLLMwg&Xtlcf!}rZfDIw&V^xev@+4A(rLG^%}TuNVuIjE{8O??@Z=5xo<>>+5KaC zO}}kexcPmY4mkzP;n8n>hB(CV?66*&_lyWPH^bzRmbPbJzkw}Hh+QdluQ|Jf;a0R` z4{5x8AL?5tQTOB@ZY(`+uZDs&hPn+mzSgS*~O`KDV=IhmMMf(sh_{{e4WO`<=|()y80>lmo!9t5qvOIoL$(s2_s zDxdRx(>5JI5PH+BfAdr$s7{aj1S`V~n3m}vBj-|r%4uZ~4V`xBz_JHEf{HdWpum)P zF}p3T(vd~aMMRq0%*nIC|Bh*sE+oqu6Dt%|yN3bh#L-`Q$?J?AL85o>9n%gSE16hgvmKd+76Ql#z+Sj@(}k?2+M^*wueRwaW@Vuo zBfgiNjTkEKHe0yf;hSoU04XZb0vohRRgAZeX{8T3hH$R;H`Z`kU6WvV&?Gg~t$d|3 zhp({UVg_>#RgIB@>Rg;IRoNa*WI(nb3T8~8Bd4clpp(kJgs9ZorWsFbQ4n|Q^bDq6 zi}P+Ol0BbMkMt7NoYukAB3yS<$p#FJMz)hEL>mXw$1>eb9hh!%|J-@7Xg5*GZ5~W1 z%63P!+>nNus#cu|Rn8szK@O*%%Xv2yo#=>B(CH`Ywb}<$&f&Y8YFre>XvmQfMfvW* z^yS#^rnWC2V^sH%67~HggPFi5VmC7|(H&y}hKeZ9_70{u!+kfEZS=@!YCDO7v~e)K zEYsc8(P1c~iiuA*A)09zOa;x5g|B1aDxOL1#tF)0AF()A!pva%%Uq%Rhcl9y{T&FPq|k^3kIU!<+`e0;boi?=5m%9V^pid+7EILMX|DE z$>1Ftd=088TGFItT%L=nyl2%q!zjFp4AW-d_?gQcUT;yaJH3=uqIg+2w=GH%@ZbIh zb?2EZ$0&TG_?M@1RhOQ$1D&Z%1P1P7LVBj_udx~hR8HgyO2xE_pYphTL|C%CwtaB* zMn;?JZPXFwi>pJ@Pr;^O2uJ`1@HqQ>NC6R)8Bt&_I^{;7c{Fw{q%ogG#88+qKo#L8 z8mf}8z!|YVQ&Z{aIV(UDC?`j)FVzbNAmALD>?=)|t!eJ7oKoTzdXiA0vOZf)U&`O< z4EoJ%PIOFsHP49Y^Yu7NF7n5H4v^?A0T!nB#Cri--E7Z8cLem^;E6yty(vQir4vjsAyl zNs42yPDuqFhg(ua)G=whnVR8T)4gCkR_}CC8Y>*cL0u^O^-)*Wk+`V~%JEdU%S_W? z&gxtkAE~Q47>xxE;iwLfy}GF*={TI!fka)@=`hnXn1gaR{734i98_bEL%1u4XRlt% z2|Nzx<&aSy=K9Ul4d%+eOwEzHvk$JZ%OPCahiR{F?Gt(&uI)od-P=cFrgAVBXOeP9 z>fH=dV~s;NHUnp`KF!EG4tHjNQBP*N&9n{X#5Mzaq`qqb8r=`!vKGZ&J=Kai4hOY( zs9)MvGbMw$qt2!usVkb=fy1RyHG;jmU^4?Y$BLhggsdy%YQKB(l&V{*ys0|{-{N*q z)ZnJ5X}aIMc|_|9una)eUAV*OC+Wm>4PO*1^6XLNjj`53jhLoTeNn5A(y6b7Dz>fa zgL5piz)#j;v4;L!86{M`WoBAknt?E)O~0QTIOc)JvCRDdww z31z!L-^c)=Ofb1!9y2IFl#LIzOR%&7jP>fjooTa&H7l5Tl5xS%-|2*f%B)nnL9JBS U4+~d}->id`ar=j_0m#$n|0xkQc>n+a delta 386 zcmZ3xow2uxaY7uU(ZqN;7DFRFlZ}ll7@73-C+}w$-~66!E9>N4>aLqJ_`8@UZ(@_= z@kmX~DamloNKMXGnp`NHJGnz>4@hG26DEPpg~F1IK#tU8PiAhgq=M)QkR-QHW^qYy zMq*J)F+@|W*aomBb{1Z+x+UTeLo!$d!2DklTfp*XSopyFMRH(sCg-q$%*~tpKz=$< zUIEDV+x${t6U*jZ>O0ssCmWq)0ck^Z Date: Thu, 4 May 2023 13:58:26 +0300 Subject: [PATCH 038/233] [#329] node: Make evacuate async Now it's possible to run evacuate shard in async. Also only one evacuate process can be in progress. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 4 + pkg/local_object_storage/engine/engine.go | 4 + pkg/local_object_storage/engine/evacuate.go | 194 ++++++++++++++++-- .../engine/evacuate_limiter.go | 178 ++++++++++++++++ .../engine/evacuate_test.go | 139 +++++++++++-- pkg/local_object_storage/shard/count.go | 31 +++ pkg/services/control/server/convert.go | 60 ++++++ pkg/services/control/server/evacuate.go | 2 +- pkg/services/control/server/evacuate_async.go | 91 +++++++- pkg/services/control/service.proto | 1 + pkg/services/control/service_grpc.pb.go | Bin 23469 -> 23641 bytes 11 files changed, 667 insertions(+), 37 deletions(-) create mode 100644 pkg/local_object_storage/engine/evacuate_limiter.go create mode 100644 pkg/local_object_storage/shard/count.go create mode 100644 pkg/services/control/server/convert.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 742f6a8f7..b84746a72 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -484,5 +484,9 @@ const ( ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" ShardGCRemoveGarbageStarted = "garbage remove started" ShardGCRemoveGarbageCompleted = "garbage remove completed" + EngineShardsEvacuationFailedToCount = "failed to get total objects count to evacuate" + EngineShardsEvacuationFailedToListObjects = "failed to list objects to evacuate" + EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" + EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 20c8a946b..b7be4756d 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -35,6 +35,7 @@ type StorageEngine struct { err error } + evacuateLimiter *evacuationLimiter } type shardWrapper struct { @@ -230,6 +231,9 @@ func New(opts ...Option) *StorageEngine { shardPools: make(map[string]util.WorkerPool), closeCh: make(chan struct{}), setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{ + guard: &sync.RWMutex{}, + }, } } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 761ed24b9..4693b2618 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -14,6 +15,9 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/atomic" "go.uber.org/zap" ) @@ -24,11 +28,23 @@ type EvacuateShardPrm struct { shardID []*shard.ID handler func(context.Context, oid.Address, *objectSDK.Object) error ignoreErrors bool + async bool } // EvacuateShardRes represents result of the EvacuateShard operation. type EvacuateShardRes struct { - count int + evacuated *atomic.Uint64 + total *atomic.Uint64 + failed *atomic.Uint64 +} + +// NewEvacuateShardRes creates new EvacuateShardRes instance. +func NewEvacuateShardRes() *EvacuateShardRes { + return &EvacuateShardRes{ + evacuated: atomic.NewUint64(0), + total: atomic.NewUint64(0), + failed: atomic.NewUint64(0), + } } // WithShardIDList sets shard ID. @@ -46,10 +62,46 @@ func (p *EvacuateShardPrm) WithFaultHandler(f func(context.Context, oid.Address, p.handler = f } -// Count returns amount of evacuated objects. +// WithAsync sets flag to run evacuate async. +func (p *EvacuateShardPrm) WithAsync(async bool) { + p.async = async +} + +// Evacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. -func (p EvacuateShardRes) Count() int { - return p.count +func (p *EvacuateShardRes) Evacuated() uint64 { + if p == nil { + return 0 + } + return p.evacuated.Load() +} + +// Total returns total count objects to evacuate. +func (p *EvacuateShardRes) Total() uint64 { + if p == nil { + return 0 + } + return p.total.Load() +} + +// Failed returns count of failed objects to evacuate. +func (p *EvacuateShardRes) Failed() uint64 { + if p == nil { + return 0 + } + return p.failed.Load() +} + +// DeepCopy returns deep copy of result instance. +func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { + if p == nil { + return nil + } + return &EvacuateShardRes{ + evacuated: atomic.NewUint64(p.evacuated.Load()), + total: atomic.NewUint64(p.total.Load()), + failed: atomic.NewUint64(p.failed.Load()), + } } const defaultEvacuateBatchSize = 100 @@ -63,15 +115,29 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (EvacuateShardRes, error) { +func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*EvacuateShardRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + shardIDs := make([]string, len(prm.shardID)) for i := range prm.shardID { shardIDs[i] = prm.shardID[i].String() } + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Evacuate", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + defer span.End() + shards, weights, err := e.getActualShards(shardIDs, prm.handler != nil) if err != nil { - return EvacuateShardRes{}, err + return nil, err } shardsToEvacuate := make(map[string]*shard.Shard) @@ -83,23 +149,91 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva } } + res := NewEvacuateShardRes() + ctx = ctxOrBackground(ctx, prm.async) + eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) + + if err != nil { + return nil, err + } + + eg.Go(func() error { + return e.evacuateShards(egCtx, shardIDs, prm, res, shards, weights, shardsToEvacuate) + }) + + if prm.async { + return nil, nil + } + + return res, eg.Wait() +} + +func ctxOrBackground(ctx context.Context, background bool) context.Context { + if background { + return context.Background() + } + return ctx +} + +func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + var err error + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + + defer func() { + span.End() + e.evacuateLimiter.Complete(err) + }() + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) - var res EvacuateShardRes + err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) + if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err)) + return err + } for _, shardID := range shardIDs { - if err = e.evacuateShard(ctx, shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) - return res, err + return err } } e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) - return res, nil + return nil +} + +func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotalObjectsCount") + defer span.End() + + for _, sh := range shardsToEvacuate { + cnt, err := sh.LogicalObjectsCount(ctx) + if err != nil { + if errors.Is(err, shard.ErrDegradedMode) { + continue + } + return err + } + res.total.Add(cnt) + } + return nil } func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", + trace.WithAttributes( + attribute.String("shardID", shardID), + )) + defer span.End() + var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -116,6 +250,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err)) return err } @@ -168,6 +303,12 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", + trace.WithAttributes( + attribute.Int("objects_count", len(toEvacuate)), + )) + defer span.End() + for i := range toEvacuate { select { case <-ctx.Done(): @@ -182,12 +323,14 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { + res.failed.Inc() continue } + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, weights, shardsToEvacuate, res) if err != nil { return err } @@ -204,15 +347,16 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - res.count++ + res.evacuated.Inc() } return nil } -func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) (bool, error) { +func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) for j := range shards { select { @@ -227,11 +371,11 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { + res.evacuated.Inc() e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr)) - res.count++ } return true, nil } @@ -239,3 +383,23 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add return false, nil } + +func (e *StorageEngine) GetEvacuationState(ctx context.Context) (*EvacuationState, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + return e.evacuateLimiter.GetState(), nil +} + +func (e *StorageEngine) EnqueRunningEvacuationStop(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + return e.evacuateLimiter.CancelIfRunning() +} diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go new file mode 100644 index 000000000..425fdc775 --- /dev/null +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -0,0 +1,178 @@ +package engine + +import ( + "context" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "golang.org/x/sync/errgroup" +) + +type EvacuateProcessState int + +const ( + EvacuateProcessStateUndefined EvacuateProcessState = iota + EvacuateProcessStateRunning + EvacuateProcessStateCompleted +) + +type EvacuationState struct { + shardIDs []string + processState EvacuateProcessState + startedAt time.Time + finishedAt time.Time + result *EvacuateShardRes + errMessage string +} + +func (s *EvacuationState) ShardIDs() []string { + if s == nil { + return nil + } + return s.shardIDs +} + +func (s *EvacuationState) Evacuated() uint64 { + if s == nil { + return 0 + } + return s.result.Evacuated() +} + +func (s *EvacuationState) Total() uint64 { + if s == nil { + return 0 + } + return s.result.Total() +} + +func (s *EvacuationState) Failed() uint64 { + if s == nil { + return 0 + } + return s.result.Failed() +} + +func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { + if s == nil { + return EvacuateProcessStateUndefined + } + return s.processState +} + +func (s *EvacuationState) StartedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.startedAt == defaultTime { + return nil + } + return &s.startedAt +} + +func (s *EvacuationState) FinishedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.finishedAt == defaultTime { + return nil + } + return &s.finishedAt +} + +func (s *EvacuationState) ErrorMessage() string { + if s == nil { + return "" + } + return s.errMessage +} + +func (s *EvacuationState) DeepCopy() *EvacuationState { + if s == nil { + return nil + } + shardIDs := make([]string, len(s.shardIDs)) + copy(shardIDs, s.shardIDs) + + return &EvacuationState{ + shardIDs: shardIDs, + processState: s.processState, + startedAt: s.startedAt, + finishedAt: s.finishedAt, + errMessage: s.errMessage, + result: s.result.DeepCopy(), + } +} + +type evacuationLimiter struct { + state EvacuationState + eg *errgroup.Group + cancel context.CancelFunc + + guard *sync.RWMutex +} + +func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { + l.guard.Lock() + defer l.guard.Unlock() + + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + default: + } + + if l.state.processState == EvacuateProcessStateRunning { + return nil, nil, logicerr.New(fmt.Sprintf("evacuate is already running for shard ids %v", l.state.shardIDs)) + } + + var egCtx context.Context + egCtx, l.cancel = context.WithCancel(ctx) + l.eg, egCtx = errgroup.WithContext(egCtx) + l.state = EvacuationState{ + shardIDs: shardIDs, + processState: EvacuateProcessStateRunning, + startedAt: time.Now().UTC(), + result: result, + } + + return l.eg, egCtx, nil +} + +func (l *evacuationLimiter) Complete(err error) { + l.guard.Lock() + defer l.guard.Unlock() + + errMsq := "" + if err != nil { + errMsq = err.Error() + } + l.state.processState = EvacuateProcessStateCompleted + l.state.errMessage = errMsq + l.state.finishedAt = time.Now().UTC() + + l.eg = nil +} + +func (l *evacuationLimiter) GetState() *EvacuationState { + l.guard.RLock() + defer l.guard.RUnlock() + + return l.state.DeepCopy() +} + +func (l *evacuationLimiter) CancelIfRunning() error { + l.guard.Lock() + defer l.guard.Unlock() + + if l.state.processState != EvacuateProcessStateRunning { + return logicerr.New("there is no running evacuation task") + } + + l.cancel() + return nil +} diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bc5b05ef0..43737e7f7 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "testing" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -21,6 +22,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" ) func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { @@ -103,14 +105,14 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, objPerShard, res.count) + require.Equal(t, uint64(objPerShard), res.Evacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -121,7 +123,7 @@ func TestEvacuateShard(t *testing.T) { // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 0, res.count) + require.Equal(t, uint64(0), res.Evacuated()) checkHasObjects(t) @@ -138,8 +140,8 @@ func TestEvacuateNetwork(t *testing.T) { var errReplication = errors.New("handler error") - acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { - var n int + acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error { + var n uint64 return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error { if n == max { return errReplication @@ -169,13 +171,13 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) prm.handler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() @@ -190,14 +192,14 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 3, res.Count()) + require.Equal(t, uint64(3), res.Evacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { @@ -205,12 +207,12 @@ func TestEvacuateNetwork(t *testing.T) { e, ids, objects := newEngineEvacuate(t, 4, 5) evacuateIDs := ids[0:3] - var totalCount int + var totalCount uint64 for i := range evacuateIDs { res, err := e.shards[ids[i].String()].List() require.NoError(t, err) - totalCount += len(res.AddressList()) + totalCount += uint64(len(res.AddressList())) } for i := range ids { @@ -223,14 +225,14 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.Count()) + require.Equal(t, totalCount-1, res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, totalCount, res.Count()) + require.Equal(t, totalCount, res.Evacuated()) }) }) } @@ -258,5 +260,114 @@ func TestEvacuateCancellation(t *testing.T) { res, err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) +} + +func TestEvacuateSingleProcess(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + eg.Go(func() error { + <-running + res, err := e.Evacuate(egCtx, prm) + require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") + require.Equal(t, uint64(0), res.Evacuated()) + close(blocker) + return nil + }) + require.NoError(t, eg.Wait()) +} + +func TestEvacuateAsync(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + st, err := e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get init state failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid init count") + require.Nil(t, st.StartedAt(), "invalid init started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + + <-running + + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get running state failed") + require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid running count") + require.NotNil(t, st.StartedAt(), "invalid running started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + expectedShardIDs := make([]string, 0, 2) + for _, id := range ids[1:2] { + expectedShardIDs = append(expectedShardIDs, id.String()) + } + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + close(blocker) + + require.Eventually(t, func() bool { + st, err = e.GetEvacuationState(context.Background()) + return st.ProcessingStatus() == EvacuateProcessStateCompleted + }, 3*time.Second, 10*time.Millisecond, "invalid final state") + + require.NoError(t, err, "get final state failed") + require.Equal(t, uint64(3), st.Evacuated(), "invalid final count") + require.NotNil(t, st.StartedAt(), "invalid final started at") + require.NotNil(t, st.FinishedAt(), "invalid final finished at") + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid final error message") + + require.NoError(t, eg.Wait()) } diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go new file mode 100644 index 000000000..b68c2f43e --- /dev/null +++ b/pkg/local_object_storage/shard/count.go @@ -0,0 +1,31 @@ +package shard + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// LogicalObjectsCount returns logical objects count. +func (s *Shard) LogicalObjectsCount(ctx context.Context) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "Shard.LogicalObjectsCount", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.GetMode().NoMetabase() { + return 0, ErrDegradedMode + } + + cc, err := s.metaBase.ObjectCounters() + if err != nil { + return 0, err + } + return cc.Logic(), nil +} diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go new file mode 100644 index 000000000..1d29ed406 --- /dev/null +++ b/pkg/services/control/server/convert.go @@ -0,0 +1,60 @@ +package control + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuationStatusResponse, error) { + shardIDs := make([][]byte, 0, len(state.ShardIDs())) + for _, shID := range state.ShardIDs() { + id, err := base58.Decode(shID) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("invalid shard id format: %s", shID)) + } + shardIDs = append(shardIDs, id) + } + var evacStatus control.GetShardEvacuationStatusResponse_Body_Status + switch state.ProcessingStatus() { + case engine.EvacuateProcessStateRunning: + evacStatus = control.GetShardEvacuationStatusResponse_Body_RUNNING + case engine.EvacuateProcessStateCompleted: + evacStatus = control.GetShardEvacuationStatusResponse_Body_COMPLETED + default: + evacStatus = control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED + } + var startedAt *control.GetShardEvacuationStatusResponse_Body_UnixTimestamp + if state.StartedAt() != nil { + startedAt = &control.GetShardEvacuationStatusResponse_Body_UnixTimestamp{ + Value: state.StartedAt().Unix(), + } + } + var duration *control.GetShardEvacuationStatusResponse_Body_Duration + if state.StartedAt() != nil { + end := time.Now().UTC() + if state.FinishedAt() != nil { + end = *state.FinishedAt() + } + duration = &control.GetShardEvacuationStatusResponse_Body_Duration{ + Seconds: int64(end.Sub(*state.StartedAt()).Seconds()), + } + } + return &control.GetShardEvacuationStatusResponse{ + Body: &control.GetShardEvacuationStatusResponse_Body{ + Shard_ID: shardIDs, + Evacuated: state.Evacuated(), + Total: state.Total(), + Failed: state.Failed(), + Status: evacStatus, + StartedAt: startedAt, + Duration: duration, + ErrorMessage: state.ErrorMessage(), + }, + }, nil +} diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index afa4011b9..fc6dd3f60 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -37,7 +37,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe resp := &control.EvacuateShardResponse{ Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.Count()), + Count: uint32(res.Evacuated()), }, } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 94ddc73d1..cdf3656e2 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -2,19 +2,96 @@ package control import ( "context" - "fmt" + "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -func (s *Server) StartShardEvacuation(context.Context, *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + var prm engine.EvacuateShardPrm + prm.WithShardIDList(s.getShardIDList(req.GetBody().GetShard_ID())) + prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) + prm.WithFaultHandler(s.replicate) + prm.WithAsync(true) + + _, err = s.s.Evacuate(ctx, prm) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StartShardEvacuationResponse{ + Body: &control.StartShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } -func (s *Server) GetShardEvacuationStatus(context.Context, *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) GetShardEvacuationStatus(ctx context.Context, req *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + state, err := s.s.GetEvacuationState(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp, err := stateToResponse(state) + if err != nil { + return nil, err + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } -func (s *Server) StopShardEvacuation(context.Context, *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + err = s.s.EnqueRunningEvacuationStop(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StopShardEvacuationResponse{ + Body: &control.StopShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index ca35ae043..a80deb2da 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -27,6 +27,7 @@ service ControlService { rpc SynchronizeTree (SynchronizeTreeRequest) returns (SynchronizeTreeResponse); // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); // StartShardEvacuation starts moving all data from one shard to the others. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 96642153363125f6062c7ca65bf27631ad2f980b..8afc6086aabe13e9d7226973518d980aac746c5e 100644 GIT binary patch delta 89 zcmZ3xo$=-l#tm9LLi+j&E~y1YsmX~YsVP KQD?83X9fW2dm5eq delta 19 bcmcb)gK_P4#tm9Ln;Uq Date: Fri, 5 May 2023 10:57:07 +0300 Subject: [PATCH 039/233] [#329] cli: Add async evacuate commands Add start, stop evacuate and evacuate status commands. Signed-off-by: Dmitrii Stepanov --- .../modules/control/evacuate_shard.go | 9 +- cmd/frostfs-cli/modules/control/evacuation.go | 296 ++++++++++++++++++ cmd/frostfs-cli/modules/control/shards.go | 3 +- pkg/services/control/rpc.go | 60 +++- 4 files changed, 354 insertions(+), 14 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/evacuation.go diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index b72ff6301..458e4cc0b 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -11,10 +11,11 @@ import ( const ignoreErrorsFlag = "no-errors" var evacuateShardCmd = &cobra.Command{ - Use: "evacuate", - Short: "Evacuate objects from shard", - Long: "Evacuate objects from shard to other shards", - Run: evacuateShard, + Use: "evacuate", + Short: "Evacuate objects from shard", + Long: "Evacuate objects from shard to other shards", + Run: evacuateShard, + Deprecated: "use frostfs-cli control shards evacuation start", } func evacuateShard(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go new file mode 100644 index 000000000..69d2dd8d4 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -0,0 +1,296 @@ +package control + +import ( + "crypto/ecdsa" + "fmt" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "github.com/spf13/cobra" + "go.uber.org/atomic" +) + +const ( + awaitFlag = "await" + noProgressFlag = "no-progress" +) + +var evacuationShardCmd = &cobra.Command{ + Use: "evacuation", + Short: "Objects evacuation from shard", + Long: "Objects evacuation from shard to other shards", +} + +var startEvacuationShardCmd = &cobra.Command{ + Use: "start", + Short: "Start evacuate objects from shard", + Long: "Start evacuate objects from shard to other shards", + Run: startEvacuateShard, +} + +var getEvacuationShardStatusCmd = &cobra.Command{ + Use: "status", + Short: "Get evacuate objects from shard status", + Long: "Get evacuate objects from shard to other shards status", + Run: getEvacuateShardStatus, +} + +var stopEvacuationShardCmd = &cobra.Command{ + Use: "stop", + Short: "Stop running evacuate process", + Long: "Stop running evacuate process from shard to other shards", + Run: stopEvacuateShardStatus, +} + +func startEvacuateShard(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + + req := &control.StartShardEvacuationRequest{ + Body: &control.StartShardEvacuationRequest_Body{ + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StartShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StartEvacuateShard(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Start evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Shard evacuation has been successfully started.") + + if awaitCompletion, _ := cmd.Flags().GetBool(awaitFlag); awaitCompletion { + noProgress, _ := cmd.Flags().GetBool(noProgressFlag) + waitEvacuateCompletion(cmd, pk, cli, !noProgress, true) + } +} + +func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.GetShardEvacuationStatusResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetEvacuateShardStatus(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Get evacuate shards status failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + printStatus(cmd, resp) +} + +func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.StopShardEvacuationRequest{ + Body: &control.StopShardEvacuationRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StopShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StopEvacuateShard(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Stop evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + waitEvacuateCompletion(cmd, pk, cli, false, false) + + cmd.Println("Evacuation stopped.") +} + +func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clientSDK.Client, printProgress, printCompleted bool) { + const statusPollingInterval = 1 * time.Second + const reportIntervalSeconds = 5 + var resp *control.GetShardEvacuationStatusResponse + reportResponse := atomic.NewPointer(resp) + pollingCompleted := make(chan struct{}) + progressReportCompleted := make(chan struct{}) + + go func() { + defer close(progressReportCompleted) + if !printProgress { + return + } + cmd.Printf("Progress will be reported every %d seconds.\n", reportIntervalSeconds) + for { + select { + case <-pollingCompleted: + return + case <-time.After(reportIntervalSeconds * time.Second): + r := reportResponse.Load() + if r == nil || r.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_COMPLETED { + continue + } + printStatus(cmd, r) + } + } + }() + + for { + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + signRequest(cmd, pk, req) + + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetEvacuateShardStatus(client, req) + return err + }) + + reportResponse.Store(resp) + + if err != nil { + commonCmd.ExitOnErr(cmd, "Failed to get evacuate status, rpc error: %w", err) + return + } + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING { + break + } + + time.Sleep(statusPollingInterval) + } + close(pollingCompleted) + <-progressReportCompleted + if printCompleted { + printCompletedStatusMessage(cmd, resp) + } +} + +func printCompletedStatusMessage(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + cmd.Println("Shard evacuation has been completed.") + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + cmd.Println(sb.String()) +} + +func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED { + cmd.Println("There is no running or completed evacuation.") + return + } + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendStatus(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + cmd.Println(sb.String()) +} + +func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetDuration() != nil { + duration := time.Second * time.Duration(resp.GetBody().GetDuration().GetSeconds()) + hour := int(duration.Seconds() / 3600) + minute := int(duration.Seconds()/60) % 60 + second := int(duration.Seconds()) % 60 + sb.WriteString(fmt.Sprintf(" Duration: %02d:%02d:%02d.", hour, minute, second)) + } +} + +func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStartedAt() != nil { + startedAt := time.Unix(resp.GetBody().GetStartedAt().GetValue(), 0).UTC() + sb.WriteString(fmt.Sprintf(" Started at: %s UTC.", startedAt.Format(time.RFC3339))) + } +} + +func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if len(resp.Body.GetErrorMessage()) > 0 { + sb.WriteString(fmt.Sprintf(" Error: %s.", resp.Body.GetErrorMessage())) + } +} + +func appendStatus(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + var status string + switch resp.GetBody().GetStatus() { + case control.GetShardEvacuationStatusResponse_Body_COMPLETED: + status = "completed" + case control.GetShardEvacuationStatusResponse_Body_RUNNING: + status = "running" + default: + status = "undefined" + } + sb.WriteString(fmt.Sprintf(" Status: %s.", status)) +} + +func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString("Shard IDs: ") + for idx, shardID := range resp.GetBody().GetShard_ID() { + shardIDStr := shard.NewIDFromBytes(shardID).String() + if idx > 0 { + sb.WriteString(", ") + } + sb.WriteString(shardIDStr) + if idx == len(resp.GetBody().GetShard_ID())-1 { + sb.WriteString(".") + } + } +} + +func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString(fmt.Sprintf(" Evacuated %d object out of %d, failed to evacuate %d objects.", + resp.GetBody().GetEvacuated(), + resp.Body.GetTotal(), + resp.Body.GetFailed())) +} + +func initControlEvacuationShardCmd() { + evacuationShardCmd.AddCommand(startEvacuationShardCmd) + evacuationShardCmd.AddCommand(getEvacuationShardStatusCmd) + evacuationShardCmd.AddCommand(stopEvacuationShardCmd) + + initControlStartEvacuationShardCmd() + initControlFlags(getEvacuationShardStatusCmd) + initControlFlags(stopEvacuationShardCmd) +} + +func initControlStartEvacuationShardCmd() { + initControlFlags(startEvacuationShardCmd) + + flags := startEvacuationShardCmd.Flags() + flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + flags.Bool(shardAllFlag, false, "Process all shards") + flags.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + flags.Bool(awaitFlag, false, "Block execution until evacuation is completed") + flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) + + startEvacuationShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 8e7ecff8c..742109673 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,13 +13,14 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(evacuateShardCmd) + shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() initControlEvacuateShardCmd() + initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 31ebfa760..3822c9f70 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,15 +8,18 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartEvacuateShard = "StartEvacuateShard" + rpcGetEvacuateShardStatus = "GetEvacuateShardStatus" + rpcStopEvacuateShardStatus = "StopEvacuateShard" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -141,6 +144,45 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return wResp.message, nil } +// StartEvacuateShard executes ControlService.StartEvacuateShard RPC. +func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { + wResp := newResponseWrapper[StartShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartEvacuateShard), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// GetEvacuateShardStatus executes ControlService.GetEvacuateShardStatus RPC. +func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { + wResp := newResponseWrapper[GetShardEvacuationStatusResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetEvacuateShardStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// StopEvacuateShard executes ControlService.StopEvacuateShard RPC. +func StopEvacuateShard(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { + wResp := newResponseWrapper[StopShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopEvacuateShardStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // FlushCache executes ControlService.FlushCache RPC. func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallOption) (*FlushCacheResponse, error) { wResp := newResponseWrapper[FlushCacheResponse]() -- 2.45.2 From 483fac03d65ba5b12e76e0d983bd887874fe1c00 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 May 2023 14:03:49 +0300 Subject: [PATCH 040/233] [#329] docs: Add shard evacuation description Signed-off-by: Dmitrii Stepanov --- docs/evacuation.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/evacuation.md diff --git a/docs/evacuation.md b/docs/evacuation.md new file mode 100644 index 000000000..7c8beb790 --- /dev/null +++ b/docs/evacuation.md @@ -0,0 +1,92 @@ +# Shard data evacuation + +## Overview + +Evacuation is the process of transferring data from one shard to another. Evacuation is used in case of problems with the shard in order to save data. + +To start the evacuation, it is necessary that the shard is in read-only mode (read more [here](./shard-modes.md)). + +First of all, by the evacuation the data is transferred to other shards of the same node; if it is not possible, then the data is transferred to other nodes. + +Only one running evacuation process is allowed on the node at a time. + +`frostfs-cli` utility is used to manage evacuation. + +## Commands + +`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. + +`frostfs-cli control shards evacuation stop` stops running evacuation process. + +`frostfs-cli control shards evacuation status` prints evacuation process status. + +See commands `--help` output for detailed description. + +## Examples + +### Set shard mode to read only +```bash +frostfs-cli control shards set-mode --mode read-only --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard mode update request successfully sent. +``` + +### Start evacuation and get status +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:05. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:13. +``` + +### Stop running evacuation process +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. + +frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Evacuation stopped. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: completed. Evacuated 31 object out of 73, failed to evacuate 0 objects. Error: context canceled. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:07. +``` + +### Start evacuation and await it completes +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await +Enter password > +Shard evacuation has been successfully started. +Progress will be reported every 5 seconds. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +``` + +### Start evacuation and await it completes without progress notifications +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await --no-progress +Enter password > +Shard evacuation has been successfully started. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. +``` \ No newline at end of file -- 2.45.2 From 9e56592be3794ccdc8d3724cc42dcc5feacdfcc3 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 5 May 2023 18:40:57 +0300 Subject: [PATCH 041/233] [#314] writecache: Simplify background workers naming Also, drop not used arg. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/flush.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 04fcccede..1e24a42ee 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -37,12 +37,12 @@ const ( func (c *cache) runFlushLoop() { for i := 0; i < c.workersCount; i++ { c.wg.Add(1) - go c.flushWorker(i) + go c.workerFlushSmall() } c.wg.Add(1) go func() { - c.flushBigObjects(context.TODO()) + c.workerFlushBig(context.TODO()) c.wg.Done() }() @@ -56,7 +56,7 @@ func (c *cache) runFlushLoop() { for { select { case <-tt.C: - c.flushDB() + c.flushSmallObjects() tt.Reset(defaultFlushInterval) case <-c.closeCh: return @@ -65,7 +65,7 @@ func (c *cache) runFlushLoop() { }() } -func (c *cache) flushDB() { +func (c *cache) flushSmallObjects() { var lastKey []byte var m []objectInfo for { @@ -148,7 +148,7 @@ func (c *cache) flushDB() { } } -func (c *cache) flushBigObjects(ctx context.Context) { +func (c *cache) workerFlushBig(ctx context.Context) { tick := time.NewTicker(defaultFlushInterval * 10) for { select { @@ -228,8 +228,8 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } -// flushWorker writes objects to the main storage. -func (c *cache) flushWorker(_ int) { +// workerFlushSmall writes small objects to the main storage. +func (c *cache) workerFlushSmall() { defer c.wg.Done() var obj *object.Object -- 2.45.2 From bf79d06f039cd1e5110b725636667b57a50dc2c2 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 5 May 2023 18:59:49 +0300 Subject: [PATCH 042/233] [#314] writecache: Do not lose small objects on disk errors Do return error if an object could not been stored on WC's disk. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/put.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index e2535d9e2..223a73bef 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -78,7 +78,7 @@ func (c *cache) putSmall(obj objectInfo) error { ) c.objCounters.IncDB() } - return nil + return err } // putBig writes object to FSTree and pushes it to the flush workers queue. -- 2.45.2 From 35c9b6b26dcbb8fe1e03f69f0f8f10262db07e29 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 May 2023 17:43:49 +0300 Subject: [PATCH 043/233] [#314] writecache: remove objects right after they are flushed Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/writecache/flush.go | 28 +-- .../writecache/flush_test.go | 99 +-------- pkg/local_object_storage/writecache/get.go | 2 - pkg/local_object_storage/writecache/init.go | 192 ------------------ .../writecache/iterate.go | 6 - pkg/local_object_storage/writecache/mode.go | 16 -- pkg/local_object_storage/writecache/put.go | 2 - .../writecache/storage.go | 46 +---- .../writecache/writecache.go | 42 ++-- 9 files changed, 25 insertions(+), 408 deletions(-) delete mode 100644 pkg/local_object_storage/writecache/init.go diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 1e24a42ee..c6c8a9465 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -78,7 +78,7 @@ func (c *cache) flushSmallObjects() { m = m[:0] c.modeMtx.RLock() - if c.readOnly() || !c.initialized.Load() { + if c.readOnly() { c.modeMtx.RUnlock() time.Sleep(time.Second) continue @@ -117,10 +117,6 @@ func (c *cache) flushSmallObjects() { var count int for i := range m { - if c.flushed.Contains(m[i].addr) { - continue - } - obj := object.New() if err := obj.Unmarshal(m[i].data); err != nil { continue @@ -157,9 +153,6 @@ func (c *cache) workerFlushBig(ctx context.Context) { if c.readOnly() { c.modeMtx.RUnlock() break - } else if !c.initialized.Load() { - c.modeMtx.RUnlock() - continue } _ = c.flushFSTree(ctx, true) @@ -187,10 +180,6 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { sAddr := addr.EncodeToString() - if _, ok := c.store.flushed.Peek(sAddr); ok { - return nil - } - data, err := f() if err != nil { c.reportFlushError("can't read a file", sAddr, err) @@ -218,9 +207,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - // mark object as flushed - c.flushed.Add(sAddr, false) - + c.deleteFromDisk(ctx, []string{sAddr}) return nil } @@ -242,9 +229,12 @@ func (c *cache) workerFlushSmall() { } err := c.flushObject(context.TODO(), obj, nil) - if err == nil { - c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true) + if err != nil { + // Error is handled in flushObject. + continue } + + c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) } } @@ -306,10 +296,6 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { cs := b.Cursor() for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) - if _, ok := c.flushed.Peek(sa); ok { - continue - } - if err := addr.DecodeString(sa); err != nil { c.reportFlushError("can't decode object address from the DB", sa, err) if ignoreErrors { diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 2cec07081..e5ca85735 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "testing" - "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -15,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -109,22 +107,9 @@ func TestFlush(t *testing.T) { require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.Flush(context.Background(), false)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("flush on moving to degraded mode", func(t *testing.T) { @@ -138,23 +123,9 @@ func TestFlush(t *testing.T) { require.NoError(t, wc.SetMode(mode.ReadOnly)) require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.SetMode(mode.Degraded)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("ignore errors", func(t *testing.T) { @@ -223,67 +194,6 @@ func TestFlush(t *testing.T) { }) }) }) - - t.Run("on init", func(t *testing.T) { - wc, bs, mb := newCache(t) - objects := []objectPair{ - // removed - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // not found - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // ok - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - } - - require.NoError(t, wc.Close()) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - - for i := range objects { - var prm meta.PutPrm - prm.SetObject(objects[i].obj) - _, err := mb.Put(context.Background(), prm) - require.NoError(t, err) - } - - var inhumePrm meta.InhumePrm - inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err := mb.Inhume(context.Background(), inhumePrm) - require.NoError(t, err) - - var deletePrm meta.DeletePrm - deletePrm.SetAddresses(objects[2].addr, objects[3].addr) - _, err = mb.Delete(context.Background(), deletePrm) - require.NoError(t, err) - - require.NoError(t, bs.SetMode(mode.ReadOnly)) - require.NoError(t, mb.SetMode(mode.ReadOnly)) - - // Open in read-only: no error, nothing is removed. - require.NoError(t, wc.Open(true)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - require.NoError(t, err, i) - } - require.NoError(t, wc.Close()) - - // Open in read-write: no error, something is removed. - require.NoError(t, wc.Open(false)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - if i < 2 { - require.ErrorAs(t, err, new(apistatus.ObjectNotFound), i) - } else { - require.NoError(t, err, i) - } - } - }) } func putObject(t *testing.T, c Cache, size int) objectPair { @@ -321,11 +231,6 @@ func newObject(t *testing.T, size int) (*object.Object, []byte) { func initWC(t *testing.T, wc Cache) { require.NoError(t, wc.Init()) - - require.Eventually(t, func() bool { - rawWc := wc.(*cache) - return rawWc.initialized.Load() - }, 100*time.Second, 1*time.Millisecond) } type dummyEpoch struct{} diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 6af1bd181..030f9b413 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -30,7 +30,6 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() - c.flushed.Get(saddr) return obj, obj.Unmarshal(value) } @@ -39,7 +38,6 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } - c.flushed.Get(saddr) return res.Object, nil } diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go deleted file mode 100644 index 2ca8cceef..000000000 --- a/pkg/local_object_storage/writecache/init.go +++ /dev/null @@ -1,192 +0,0 @@ -package writecache - -import ( - "context" - "errors" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" - "go.uber.org/zap" -) - -func (c *cache) initFlushMarks(ctx context.Context) { - var localWG sync.WaitGroup - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.fsTreeFlushMarkUpdate(ctx) - }() - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.dbFlushMarkUpdate(ctx) - }() - - c.initWG.Add(1) - c.wg.Add(1) - go func() { - defer c.wg.Done() - defer c.initWG.Done() - - localWG.Wait() - - select { - case <-c.stopInitCh: - return - case <-c.closeCh: - return - default: - } - - c.initialized.Store(true) - }() -} - -var errStopIter = errors.New("stop iteration") - -func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) - - var prm common.IteratePrm - prm.LazyHandler = func(addr oid.Address, _ func() ([]byte, error)) error { - select { - case <-c.closeCh: - return errStopIter - case <-c.stopInitCh: - return errStopIter - default: - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - var prm common.DeletePrm - prm.Address = addr - - _, err := c.fsTree.Delete(ctx, prm) - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(addr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("fstree DELETE"), - ) - } - } - } - return nil - } - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - _, _ = c.fsTree.Iterate(prm) - - c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) -} - -func (c *cache) dbFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - var m []string - var indices []int - var lastKey []byte - var batchSize = flushBatchSize - for { - select { - case <-c.closeCh: - return - case <-c.stopInitCh: - return - default: - } - - m = m[:0] - indices = indices[:0] - - // We put objects in batches of fixed size to not interfere with main put cycle a lot. - _ = c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - cs := b.Cursor() - for k, _ := cs.Seek(lastKey); k != nil && len(m) < batchSize; k, _ = cs.Next() { - m = append(m, string(k)) - } - return nil - }) - - var addr oid.Address - for i := range m { - if err := addr.DecodeString(m[i]); err != nil { - continue - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - indices = append(indices, i) - } - } - } - - if len(m) == 0 { - break - } - - err := c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - for _, j := range indices { - if err := b.Delete([]byte(m[j])); err != nil { - return err - } - } - return nil - }) - if err == nil { - for _, j := range indices { - storagelog.Write(c.log, - zap.String("address", m[j]), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - } - } - lastKey = append([]byte(m[len(m)-1]), 0) - } - - c.log.Info(logs.WritecacheFinishedUpdatingFlushMarks) -} - -// flushStatus returns info about the object state in the main storage. -// First return value is true iff object exists. -// Second return value is true iff object can be safely removed. -func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) { - var existsPrm meta.ExistsPrm - existsPrm.SetAddress(addr) - - _, err := c.metabase.Exists(ctx, existsPrm) - if err != nil { - needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)) - return needRemove, needRemove - } - - var prm meta.StorageIDPrm - prm.SetAddress(addr) - - mRes, _ := c.metabase.StorageID(ctx, prm) - res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) - return err == nil && res.Exists, false -} diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 228dd2597..ebe979520 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -41,9 +41,6 @@ func (c *cache) Iterate(prm IterationPrm) error { err := c.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) return b.ForEach(func(k, data []byte) error { - if _, ok := c.flushed.Peek(string(k)); ok { - return nil - } return prm.handler(data) }) }) @@ -54,9 +51,6 @@ func (c *cache) Iterate(prm IterationPrm) error { var fsPrm common.IteratePrm fsPrm.IgnoreErrors = prm.ignoreErrors fsPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - if _, ok := c.flushed.Peek(addr.EncodeToString()); ok { - return nil - } data, err := f() if err != nil { if prm.ignoreErrors { diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 14f8af49e..ca6faff4c 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -37,22 +37,6 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() - if !c.initialized.Load() { - close(c.stopInitCh) - - c.initWG.Wait() - c.stopInitCh = make(chan struct{}) - - defer func() { - if err == nil && !turnOffMeta { - c.initFlushMarks(ctx) - } - }() - } - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - if turnOffMeta && !c.mode.NoMetabase() { err = c.flush(ctx, true) if err != nil { diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 223a73bef..04d818b31 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -37,8 +37,6 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro defer c.modeMtx.RUnlock() if c.readOnly() { return common.PutRes{}, ErrReadOnly - } else if !c.initialized.Load() { - return common.PutRes{}, ErrNotInitialized } sz := uint64(len(prm.RawData)) diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index aeae752e3..c06d16c0b 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -13,8 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - lru "github.com/hashicorp/golang-lru/v2" - "github.com/hashicorp/golang-lru/v2/simplelru" "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -22,19 +20,7 @@ import ( // store represents persistent storage with in-memory LRU cache // for flushed items on top of it. type store struct { - maxFlushedMarksCount int - maxRemoveBatchSize int - - // flushed contains addresses of objects that were already flushed to the main storage. - // We use LRU cache instead of map here to facilitate removing of unused object in favour of - // frequently read ones. - // MUST NOT be used inside bolt db transaction because it's eviction handler - // removes untracked items from the database. - flushed simplelru.LRUCache[string, bool] - db *bbolt.DB - - dbKeysToRemove []string - fsKeysToRemove []string + db *bbolt.DB } const dbName = "small.bolt" @@ -73,35 +59,9 @@ func (c *cache) openStore(readOnly bool) error { return fmt.Errorf("could not open FSTree: %w", err) } - // Write-cache can be opened multiple times during `SetMode`. - // flushed map must not be re-created in this case. - if c.flushed == nil { - c.flushed, _ = lru.NewWithEvict[string, bool](c.maxFlushedMarksCount, c.removeFlushed) - } - - c.initialized.Store(false) - return nil } -// removeFlushed removes an object from the writecache. -// To minimize interference with the client operations, the actual removal -// is done in batches. -// It is not thread-safe and is used only as an evict callback to LRU cache. -func (c *cache) removeFlushed(key string, value bool) { - fromDatabase := value - if fromDatabase { - c.dbKeysToRemove = append(c.dbKeysToRemove, key) - } else { - c.fsKeysToRemove = append(c.fsKeysToRemove, key) - } - - if len(c.dbKeysToRemove)+len(c.fsKeysToRemove) >= c.maxRemoveBatchSize { - c.dbKeysToRemove = c.deleteFromDB(c.dbKeysToRemove) - c.fsKeysToRemove = c.deleteFromDisk(c.fsKeysToRemove) - } -} - func (c *cache) deleteFromDB(keys []string) []string { if len(keys) == 0 { return keys @@ -133,7 +93,7 @@ func (c *cache) deleteFromDB(keys []string) []string { return keys[:len(keys)-errorIndex] } -func (c *cache) deleteFromDisk(keys []string) []string { +func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { if len(keys) == 0 { return keys } @@ -147,7 +107,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { continue } - _, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index bdcc9bbf6..83ecf219c 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,7 +5,6 @@ import ( "os" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -13,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -52,11 +50,8 @@ type cache struct { // mtx protects statistics, counters and compressFlags. mtx sync.RWMutex - mode mode.Mode - initialized atomic.Bool - stopInitCh chan struct{} // used to sync initWG initialisation routines and _only_ them - initWG sync.WaitGroup // for initialisation routines only - modeMtx sync.RWMutex + mode mode.Mode + modeMtx sync.RWMutex // compressFlags maps address of a big object to boolean value indicating // whether object should be compressed. @@ -96,9 +91,8 @@ var ( // New creates new writecache instance. func New(opts ...Option) Cache { c := &cache{ - flushCh: make(chan *object.Object), - mode: mode.ReadWrite, - stopInitCh: make(chan struct{}), + flushCh: make(chan *object.Object), + mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), options: options{ @@ -117,12 +111,6 @@ func New(opts ...Option) Cache { opts[i](&c.options) } - // Make the LRU cache contain which take approximately 3/4 of the maximum space. - // Assume small and big objects are stored in 50-50 proportion. - c.maxFlushedMarksCount = int(c.maxCacheSize/c.maxObjectSize+c.maxCacheSize/c.smallObjectSize) / 2 * 3 / 4 - // Trigger the removal when the cache is 7/8 full, so that new items can still arrive. - c.maxRemoveBatchSize = c.maxFlushedMarksCount / 8 - return c } @@ -153,31 +141,27 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init") - defer span.End() - - c.initFlushMarks(ctx) c.runFlushLoop() return nil } // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { - // Finish all in-progress operations. - if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { - return err - } - + // We cannot lock mutex for the whole operation duration + // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. + c.modeMtx.Lock() if c.closeCh != nil { close(c.closeCh) } + c.mode = mode.DegradedReadOnly // prevent new operations from being processed + c.modeMtx.Unlock() + c.wg.Wait() - if c.closeCh != nil { - c.closeCh = nil - } - c.initialized.Store(false) + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.closeCh = nil var err error if c.db != nil { err = c.db.Close() -- 2.45.2 From e3ad3c296510ab08132c73d372de9173a852eb52 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 18 May 2023 13:08:05 +0300 Subject: [PATCH 044/233] [#283] cli: Move control healthcheck command under control ir Signed-off-by: Alejandro Lopez --- .../modules/control/healthcheck.go | 41 ++++------------- cmd/frostfs-cli/modules/control/ir.go | 2 + .../modules/control/ir_healthcheck.go | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/ir_healthcheck.go diff --git a/cmd/frostfs-cli/modules/control/healthcheck.go b/cmd/frostfs-cli/modules/control/healthcheck.go index 8d18a5c11..097fba540 100644 --- a/cmd/frostfs-cli/modules/control/healthcheck.go +++ b/cmd/frostfs-cli/modules/control/healthcheck.go @@ -1,15 +1,10 @@ package control import ( - "crypto/ecdsa" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" - ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -19,8 +14,8 @@ const ( var healthCheckCmd = &cobra.Command{ Use: "healthcheck", - Short: "Health check of the FrostFS node", - Long: "Health check of the FrostFS node. Checks storage node by default, use --ir flag to work with Inner Ring.", + Short: "Health check for FrostFS storage nodes", + Long: "Health check for FrostFS storage nodes.", Run: healthCheck, } @@ -29,18 +24,18 @@ func initControlHealthCheckCmd() { flags := healthCheckCmd.Flags() flags.Bool(healthcheckIRFlag, false, "Communicate with IR node") + _ = flags.MarkDeprecated(healthcheckIRFlag, "for health check of inner ring nodes, use the 'control ir healthcheck' command instead.") } -func healthCheck(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - cli := getClient(cmd, pk) - +func healthCheck(cmd *cobra.Command, args []string) { if isIR, _ := cmd.Flags().GetBool(healthcheckIRFlag); isIR { - healthCheckIR(cmd, pk, cli) + irHealthCheck(cmd, args) return } + pk := key.Get(cmd) + cli := getClient(cmd, pk) + req := new(control.HealthCheckRequest) req.SetBody(new(control.HealthCheckRequest_Body)) @@ -59,23 +54,3 @@ func healthCheck(cmd *cobra.Command, _ []string) { cmd.Printf("Network status: %s\n", resp.GetBody().GetNetmapStatus()) cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) } - -func healthCheckIR(cmd *cobra.Command, key *ecdsa.PrivateKey, c *client.Client) { - req := new(ircontrol.HealthCheckRequest) - - req.SetBody(new(ircontrol.HealthCheckRequest_Body)) - - err := ircontrolsrv.SignMessage(key, req) - commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) - - var resp *ircontrol.HealthCheckResponse - err = c.ExecRaw(func(client *rawclient.Client) error { - resp, err = ircontrol.HealthCheck(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) -} diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index ae3f8502c..396d5d0a5 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -11,7 +11,9 @@ var irCmd = &cobra.Command{ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) irCmd.AddCommand(removeNodeCmd) + irCmd.AddCommand(irHealthCheckCmd) initControlIRTickEpochCmd() initControlIRRemoveNodeCmd() + initControlIRHealthCheckCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_healthcheck.go b/cmd/frostfs-cli/modules/control/ir_healthcheck.go new file mode 100644 index 000000000..e70538ce2 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_healthcheck.go @@ -0,0 +1,44 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var irHealthCheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Health check for FrostFS inner ring nodes", + Long: "Health check for FrostFS inner ring nodes.", + Run: irHealthCheck, +} + +func initControlIRHealthCheckCmd() { + initControlFlags(irHealthCheckCmd) +} + +func irHealthCheck(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + cli := getClient(cmd, pk) + + req := new(ircontrol.HealthCheckRequest) + + req.SetBody(new(ircontrol.HealthCheckRequest_Body)) + + err := ircontrolsrv.SignMessage(pk, req) + commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) + + var resp *ircontrol.HealthCheckResponse + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.HealthCheck(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) +} -- 2.45.2 From 731bf5d0ee6683e4bf624813356039b96e915d12 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 19 May 2023 16:46:01 +0300 Subject: [PATCH 045/233] [#380] node: go version up Signed-off-by: Dmitrii Stepanov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .docker/Dockerfile.storage-testnet | 2 +- CHANGELOG.md | 1 + README.md | 2 +- go.mod | 2 +- scripts/export-metrics/main.go | 3 +-- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index 09d66e642..f0075c7ab 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index 010616a17..e22762b7b 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.19 +FROM golang:1.20 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index c706359b3..8fad863be 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 9f8e72386..0b01ea7e4 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index 39eb19559..dd9ff2684 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage-testnet b/.docker/Dockerfile.storage-testnet index 908ff0aad..28bbf7a1b 100644 --- a/.docker/Dockerfile.storage-testnet +++ b/.docker/Dockerfile.storage-testnet @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/CHANGELOG.md b/CHANGELOG.md index 97feb2245..337a03794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Changelog for FrostFS Node - `github.com/spf13/cobra` to `v1.7.0` - `github.com/multiformats/go-multiaddr` to `v0.9.0` - `go.uber.org/atomic` to `v1.11.0` +- Minimum go version to v1.19 ### Updating from v0.36.0 diff --git a/README.md b/README.md index 81701c441..f228cd426 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.18+ and `make`: +To make all binaries you need Go 1.19+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index 2e9286a05..5c6972f4b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.18 +go 1.19 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index ac6e786ab..694eea38b 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -4,7 +4,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "os" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" @@ -49,7 +48,7 @@ func main() { os.Exit(1) } - if err := ioutil.WriteFile(filename, data, 0644); err != nil { + if err := os.WriteFile(filename, data, 0644); err != nil { fmt.Fprintf(os.Stderr, "Could write to file: %v\n", err) os.Exit(1) } -- 2.45.2 From 7eb8fa6350b42fb8e762a61807a583c9fafe81d3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 19 May 2023 13:00:27 +0300 Subject: [PATCH 046/233] [#381] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 11 ++++++++- go.mod | 65 ++++++++++++++++++++++++++------------------------- go.sum | Bin 96389 -> 97269 bytes 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 337a03794..71f1a98b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,14 +22,23 @@ Changelog for FrostFS Node ### Removed ### Updated -- `paulmach/orb` to v0.9.1 +- `neo-go` to `v0.101.1` +- `google.golang.org/grpc` to `v1.55.0` +- `paulmach/orb` to `v0.9.2` - `github.com/nats-io/nats.go` to `v1.25.0` - `golang.org/x/sync` to `v0.2.0` - `golang.org/x/term` to `v0.8.0` - `github.com/spf13/cobra` to `v1.7.0` +- `github.com/panjf2000/ants/v2` `v2.7.4` - `github.com/multiformats/go-multiaddr` to `v0.9.0` +- `github.com/hashicorp/golang-lru/v2` to `v2.0.2` - `go.uber.org/atomic` to `v1.11.0` - Minimum go version to v1.19 +- `github.com/prometheus/client_golang` to `v1.15.1` +- `github.com/prometheus/client_model` to `v0.4.0` +- `go.opentelemetry.io/otel` to `v1.15.1` +- `go.opentelemetry.io/otel/trace` to `v1.15.1` +- `github.com/spf13/cast` to `v1.5.1` ### Updating from v0.36.0 diff --git a/go.mod b/go.mod index 5c6972f4b..ed92cc7b6 100644 --- a/go.mod +++ b/go.mod @@ -12,32 +12,32 @@ require ( github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/klauspost/compress v1.16.5 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.25.0 - github.com/nspcc-dev/neo-go v0.100.1 + github.com/nspcc-dev/neo-go v0.101.1 github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.4.0 - github.com/paulmach/orb v0.9.1 - github.com/prometheus/client_golang v1.15.0 - github.com/prometheus/client_model v0.3.0 - github.com/spf13/cast v1.5.0 + github.com/panjf2000/ants/v2 v2.7.4 + github.com/paulmach/orb v0.9.2 + github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_model v0.4.0 + github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 go.etcd.io/bbolt v1.3.6 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.15.1 + go.opentelemetry.io/otel/trace v1.15.1 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 golang.org/x/term v0.8.0 - google.golang.org/grpc v1.53.0 + google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,29 +45,29 @@ require ( require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.0 // indirect + github.com/holiman/uint256 v1.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -84,31 +84,32 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/common v0.43.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/twmb/murmur3 v1.1.5 // indirect - github.com/urfave/cli v1.22.5 // indirect - go.mongodb.org/mongo-driver v1.11.4 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect - go.opentelemetry.io/otel/sdk v1.14.0 // indirect + github.com/twmb/murmur3 v1.1.7 // indirect + github.com/urfave/cli v1.22.13 // indirect + go.mongodb.org/mongo-driver v1.11.6 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect + go.opentelemetry.io/otel/sdk v1.15.1 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) retract ( diff --git a/go.sum b/go.sum index 29fef2509c848412ace06c3f171feb15d77f35e7..913a57613a0848cdd73e89062df5411012606770 100644 GIT binary patch delta 4157 zcmai1+t1|mdDZ~C6i6V1g#{5xLJ$hFSf4#MC=EW1#~#~b&x}3ek)o~-W6#(#K8)?L z$F0&P=FA-^WTHt5p31 zp67ku)ARei-~8c~AN}~so4@-=`?W`|z69SuZopuW+}I`F7uWEMY`w5Vbib7bjj?0Pgt;3cI9J{Ux;`~~vy|BLJNBrCh*Js<;(Z>+v26V_Rtc<%?J@Ki1Iv9kN9LBU-4pkMz>KAEKJ#I!&OKwwO@Xu{RaJ+9(MXS`o+6jhaXYfB>3V1z5!pP4ti+HjyP?u#s_h}f)1vUsv3%9 zE@_||XV%B51|CrY%J^0vUOW9cA3ty*{zwap{<;)~i>)h<`GScDSk)is6^#&85~^%9 zwrGAWhq`ixY#-D+8SPHvPjBk=gY7ThCzkGLx>KHy~|IB9F->{!SIvV7>qMW=qr1+HDG~w3E$}*wX zT;sY<9*t$9;8;9VrpFp>>h`a0KHt7<-}~Yj^intbsi#)voMDE-hOKRCOXp3-N{Zvp z#HrLz6u(GGzz|G`Jbi5cPY*he%T>F`+vBPWCd_6LNK=&MhU!`>#9>q}QEoZFiz?6l#{!E1gK;!RWJb0?HIBn=Q!E_sOR63$cUmJGv-wK z{b&05`&z9L+7GKM?aR$WFnm6fKpi{FDZk>Ju^8asjwhr6R1q=dl@oZ2g7clTiRWfz zlt%lx>ajB<31gQED0!P=C65rK_oks@sk^EHg4nUM%Sf>!O;9Ku zsKZ_kj}>`Z&#QhI#S4fS0OnN5M+b4Erk3EFGY1iClZ7DMGTPU^eZAwj+Wz9JkDq?@ z%5x7O{Tt^s*h!hf6^a`zx5ZJ$a5BwESQJ!$!>#$CS>QRiU5Ax67nN4|?lV*eyQE+w zGCND4bbWyvNY)Ix{m~^m1LLVf?Co_ooBS}^vr3?l?Vo)2>5I;%pfxvGp#^x{vChmvT-NeK(xLpp~U8Y*Qf;s$36;9JhNb_Jz3ZLx{_iBK(-9Z)cb+kF+6_y z>Fd96_pfXY30TIPeAQVZ)IEEb2_>QmF_T|+#=0zGa~sbXWiBS3AB3c=kM%(!&XRg7 z6}9`-`-4rfE}a{0vIF;@CdDpBik$hiL3(pS^D++U2cbeLW~7!IH>kz1PIuU-Ut?iP z=9~Qgp}UmyKt-pAy#g$~5GKN-?C-{rqB`4!;wCQ1`Lp4^WHrOBNwNk1;^Q`WgKL#H zuA?w?QTCZ~oQ;R*0OS~$XeOp>9rr6`NC@$AxvzASPONakERXYn-u~o`ublqj4dov2 zd{ZYtG%o!0T<3|IsW%pYas5h?nIQ5^J7@VpJvA7MvBYhw|MfN6we#Za)#)bhl6;5K zyiiSxaoQ~Q{YeIAR3Irbh`64h(+NSTqeh3Jk&2P0@0_%ITk37D{qS!eh0rszm}qE$ zq}8DBD!jwOURE$fnuKV)sX!`-O^gi#JelJ`VYT#IkG1LFJV22*&`a_22Hfr}alGKb z4$kdHVVKFsB{GSFe6;uY5Gxlo4%wwv`QoM@tDr#x*K8@CZoS``&xDPh2XVK+S@@^747Oz=|D5_CO zeehWO(hsh+*WZ5x`6T|L`xg~z<2lF-D-}46N@Uk3R#SzOf^01T#Hfj~Kxb(AQ}8=P zl;fG%#{qfVn829Z$)vrW$n$7GqbuE=vWQLRPA!(0S4N+Rc&bhQ>B?;p#W7jg#+sb) zBm3A#IMd3fJY$(K3HZWTT>+M*+2cBp6aF&%4iQ6R8t+40hpEwm;@L3O>(b8$0J=#y zv=OJ8vbD>Zwk(n06Bg*UfK>Do0T=1zph+p-*h-BzjZ&jZZG)y`4w#g1%*ZpE74_TD zJA$W)I%2W`4#X3Xh*n4<%(!sqNBKSK4EyMeg6m7Zwo-QGA_sX z)QMDFS+h<#WIeQCiEQuMAy9I3v;Z^4z!^Hf48KD}Mgp_74dC)@9j>MZ$;>IVG+4aC zL4b>`p)z8Wbi(H420CZ?-ami#wg7xEngDqeq|ITi=|hqC-Ab`Smgz1MdEVq`In72J zz2Rr#ZRj1r$-PnKb$Yg9T29G+I75q-{}{>wwRgO>|A zAvQyaZ#aLivXBhz)KDzM6e6Zao9X(97ig_)%LD|#&+k) zE9o$#j&)H=wC`f(9H)Z=wFkR3y58RW;K9?wAN=~~cpS!I3h&OU7kt!%Y|5o<+;z_u zbH`5N*~&CRcfa0Fc4NpKkv0(#lf6&O2GqK!5+s$(X($@)_y7GV_3SkJSzMPq!i41_ z5(1(xBo?B=bS$fVQkVBl9mz+j>Q5vG4w~_}p{f)AAHUb#X1Cz0J@KuNuk=1LIjsBJ z;dmdTUhr(YkKr!VCs*elGD@qc8IJftpP`m@0F(1(Z1=rT0){ruHN{#vE-l^XW8~Ja z&3j)_Q0QlKU6LA)cyJ@yauy?wUM-k_^x*)pwrgUKD=NaE}U zdheloyLO%*kmxuh5wI9c*_E-M%mNVS!M+D}9e|6F7#(4nq041?!0q7Ht9tLzTi^0} zKi5su<^RUU;uj+!;+b^-YvEEqBo=RKGcFn?FzAG{*so@*&KqVHdCL}h*KWPC?%n^^ zS696!KJ(TeX1&do?wUO>j#Xyaq_neKF#9lLg7J>3w<4sBj02{#I2P(L!>;F!5R12d zc<4RVowJu8WoSH@q8ooxu~KdkZQjtCw+57|A5c; z_kDib^F06k=3SqCT=oIz>Zv&o!0oWJ_bTpqpr6P$>u~Dq3OfpFHto2RaO&c>RxLFK@DP4bB#B8X!3F zSPh>+XAl^z&z5Wdq+5&@NJiKQVH<#>dHQ5Xl(FisFfxv*v?TJ4zNnYE#WP;}I(ohR zD*7M-oxvy6mP?IOWiHmV0ga@l(CF)Qq$)K#o=}I{Z}CL6qoN0bZ5eeNfYPuhNWO$ z9zn8Aqk- zW}W)7R||uU0_>n^=(TVD*}T2x{>25?bwqF1?M--5@{J~LdZAXYz_C9CM}jX)Sc77l zk&p4sDvyHdCobk%Ee$W6B0e8 z%4NBmm!i`CEQk*O9=>w*PUwt2@SF6Wvr`G=B~9ZYnuj|-^NESRtPEB(X5c8UvdUW6 zIaRO{IP!Zy`~CX;?eF*3+8cZE@ZzVx@&No+*EYuj&PHQ@uK?Q2aPeiC)8Gor?PRA* zdETr7N-=U;?QYv|eEy;K3(wzsThpqmjO4L^mb#u-MCGxR<N}I4&6& zjZYPL^^5r7o-aQ69t1nPnBrhL8w^K*YAWN|DjyG~;~JY-{cSQDn@W!fH_n6@26*hK zUh97K!}JlmG%g{*L#ku&g}`PwLyk<)Nx&tIxTU$(QXA5wnBR?6KF65W{n~@{I-fmO zAMEudmSk$|vArRzTNaRy>bbUs0x&w!SO1Sb0AG9U?we}} z57|ht*U<`!c|UGqdA23vUSaO4a8TNGx@c-&4fzEhEA8XoeClra493r(!zW)L?m*C+ zIhb}MffqNnFMyLF&%s@I9XZXG+w{_HhK60G-uUwoX$4!Xeg1_%rjOXA9LStt4jgGs z=4pe*QQZvy&t0qlE=T=g6m@aGPD+m4(!lm;;Cicl=bi)q?WgWQj;(m=px&Set)9Kc z`kASSBJM@NK%1ID)z{ZGEcw)47*jzY!S?9@7d(v}*-I97-R;H*@y?yq(R}YWAh~3#r-+S$13_yNv7?iJr0bp-oi@cz`EMU@|LgCBmiW;FhyVHEjXN#`=Cmg3beLB1zH22V;b-N7=`|@% zh5bUGShI12>XFfGqO|>wAAhL*_K&&tnOCm0Uw_%U;1}wKE*Q}oAe+#aC=(FIQkk&5 zR05Fo81n(8CpIuUu-4P|?3E9;Z@$d6=1-nEJp7Z_?z#}b+$M2T27=L}eW8(&0qSg; z#l-c%TuJi2rZ{j}CDqqLmG#D?tlkG7xt$NS!n9$KO!3U$6F6k}tcSlHil@tiC~A+yk40*;bpxPl-wUf)7* zTOw0lAbVY9>{|_Al-gE=5NZk)MhGxH@lR=17Oo3EERPAMV zr?Fj2&BR@(fi6o98J{b35t3%Xcyz*U3o7O_)~GQ^)8q-$U1}@XG0TQvMJt_B+yI>B z9)^};6_L=9MqBOYZ=cxBfQA}d7OGUVn+=RoOBKS@`JOH-)BxS8qeh!jg()6i8E>Jt z1;@7-%-MuaM@~LVstsWSB#s6&VTQxl#U-!Mj_--GS}c_IxnF$z)`^RrOhf%d(?F~_ z*2Y@k`w5fEa^c{B(s)c{(X`GLJHew)*lj`AY_XUgUjw$E4eUmz1FFcK-bNw1T`JsQ zu+i@(GSegW@b+l?;in5fz11nFH8teeEaS)AhGO7ErfXuLNQ7?(g}VVb-s7A)Db_;0 z6ZZBAXD;EFMtruFSbUP?sz8i5R#eT}#6(#vrz1=Pj1gC?lDKXhmXS}r`tZlg-1UOX zHntg}Eb9b=ORi{`zBM;FGYi#1n@f$4Qei{kSu^nX4R|5IzC+-W5z4JsBY6O@0!cQi zJaq#saFISYBIXkyTJPhPlQdwyr7xQ9!S)xgzW4CwuYTrzJp{pE5=BtxW8U%NM`oCZ zAmPOuyI*@`RBToV?Da)<$(TDxp3j%r5T;~jIo+ibT*=6u123ia+AkkfQRokAU2@6p zq&gV54Zu%gn1h<36D<2i#DX+#?5j$XH~46d?N)2sYajphjuwlfU0vqcTuE&~c0C@zk$zH&8XP}%v=XM@> zNpL&YuE1|yz%o-KenEOm-Th&u7aSa+PNhx*ZFW)bZ1#9CF=Pv?aqVe&$^u} zPvh^rhP9QHr_ekz`~*AxOlWDk=BGOn0gVMm0?D)xw@$Rxl~J}mf0ggtf1a&6S6}*F z+_`>lhdlqwqNBd_K-HPwg}`UzDN8~$R^p{!R09f_&xxtSb26pQ0Haa1uFphXzX@$| zClD*>+}(E`rH|aDgp}!^r Date: Fri, 19 May 2023 13:08:13 +0300 Subject: [PATCH 047/233] [#381] go.mod: Update bbolt Adopt new `ForEachBucket` function where possible. Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | Bin 97269 -> 97348 bytes .../metabase/iterators.go | 6 +----- pkg/local_object_storage/metabase/select.go | 6 +----- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f1a98b4..f79940f76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Changelog for FrostFS Node - `neo-go` to `v0.101.1` - `google.golang.org/grpc` to `v1.55.0` - `paulmach/orb` to `v0.9.2` +- `go.etcd.io/bbolt` to `v1.3.7` - `github.com/nats-io/nats.go` to `v1.25.0` - `golang.org/x/sync` to `v0.2.0` - `golang.org/x/term` to `v0.8.0` diff --git a/go.mod b/go.mod index ed92cc7b6..d90f45766 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.3 - go.etcd.io/bbolt v1.3.6 + go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.15.1 go.opentelemetry.io/otel/trace v1.15.1 go.uber.org/atomic v1.11.0 diff --git a/go.sum b/go.sum index 913a57613a0848cdd73e89062df5411012606770..648afba0e2cb76ed7094c6bf088c1dad3adda763 100644 GIT binary patch delta 166 zcmezRo%P5M)(ze#CQm+LU7uQ#oT8VRub-5ZpHrexW~gVZXReT8XqBa1<>jB3=bGiF zpIzyZwJuZPVXeV}-5ehSDqKTF^2 ya?1#d3e&XGf=UBV@8aCj!myHjVRx6xisD3 Date: Fri, 19 May 2023 18:06:20 +0300 Subject: [PATCH 048/233] [#381] *: Move to sync/atomic Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/evacuation.go | 4 +-- cmd/frostfs-node/config.go | 14 ++++++---- cmd/frostfs-node/netmap.go | 18 ++++++------ go.mod | 2 +- pkg/innerring/indexer_test.go | 6 ++-- pkg/innerring/innerring.go | 4 +-- pkg/innerring/state.go | 4 +-- .../blobovnicza/blobovnicza.go | 2 +- pkg/local_object_storage/blobovnicza/sizes.go | 2 +- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/engine.go | 6 ++-- .../engine/engine_test.go | 4 +-- pkg/local_object_storage/engine/evacuate.go | 28 +++++++++++-------- pkg/local_object_storage/engine/shards.go | 4 +-- .../internal/testutil/generators.go | 6 ++-- pkg/local_object_storage/metabase/put_test.go | 13 +++++---- .../shard/control_test.go | 2 +- .../writecache/flush_test.go | 4 +-- pkg/local_object_storage/writecache/state.go | 11 ++++---- pkg/morph/client/client.go | 2 +- pkg/util/worker_pool.go | 3 +- 21 files changed, 77 insertions(+), 64 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 69d2dd8d4..665bcf85f 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "fmt" "strings" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -13,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" - "go.uber.org/atomic" ) const ( @@ -134,7 +134,7 @@ func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clien const statusPollingInterval = 1 * time.Second const reportIntervalSeconds = 5 var resp *control.GetShardEvacuationStatusResponse - reportResponse := atomic.NewPointer(resp) + reportResponse := new(atomic.Pointer[control.GetShardEvacuationStatusResponse]) pollingCompleted := make(chan struct{}) progressReportCompleted := make(chan struct{}) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c45d44fd2..fe2b5ad35 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -11,7 +11,7 @@ import ( "path/filepath" "strings" "sync" - atomicstd "sync/atomic" + "sync/atomic" "syscall" "time" @@ -68,7 +68,6 @@ import ( neogoutil "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -374,7 +373,7 @@ type shared struct { ownerIDFromKey user.ID // user ID calculated from key // current network map - netMap atomicstd.Value // type netmap.NetMap + netMap atomic.Value // type netmap.NetMap netMapSource netmapCore.Source cnrClient *containerClient.Client @@ -582,6 +581,9 @@ func initCfg(appCfg *config.Config) *cfg { } func initInternals(appCfg *config.Config, log *logger.Logger) internals { + var healthStatus atomic.Int32 + healthStatus.Store(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)) + return internals{ done: make(chan struct{}), appCfg: appCfg, @@ -589,7 +591,7 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { log: log, wg: new(sync.WaitGroup), apiVersion: version.Current(), - healthStatus: atomic.NewInt32(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)), + healthStatus: &healthStatus, } } @@ -627,12 +629,14 @@ func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) c netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) + var reBootstrapTurnedOff atomic.Bool + reBootstrapTurnedOff.Store(relayOnly) return cfgNetmap{ scriptHash: contractsconfig.Netmap(appCfg), state: netState, workerPool: netmapWorkerPool, needBootstrap: !relayOnly, - reBoostrapTurnedOff: atomic.NewBool(relayOnly), + reBoostrapTurnedOff: &reBootstrapTurnedOff, } } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 58e3cb2f2..c6623a385 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -19,7 +20,6 @@ import ( netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -27,7 +27,7 @@ import ( type networkState struct { epoch *atomic.Uint64 - controlNetStatus atomic.Value // control.NetmapStatus + controlNetStatus atomic.Int32 // control.NetmapStatus nodeInfo atomic.Value // *netmapSDK.NodeInfo @@ -35,13 +35,11 @@ type networkState struct { } func newNetworkState() *networkState { - var nmStatus atomic.Value - nmStatus.Store(control.NetmapStatus_STATUS_UNDEFINED) - - return &networkState{ - epoch: atomic.NewUint64(0), - controlNetStatus: nmStatus, + ns := &networkState{ + epoch: new(atomic.Uint64), } + ns.controlNetStatus.Store(int32(control.NetmapStatus_STATUS_UNDEFINED)) + return ns } func (s *networkState) CurrentEpoch() uint64 { @@ -91,11 +89,11 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { // calls will process this value to decide what status node should set in the // network. func (s *networkState) setControlNetmapStatus(st control.NetmapStatus) { - s.controlNetStatus.Store(st) + s.controlNetStatus.Store(int32(st)) } func (s *networkState) controlNetmapStatus() (res control.NetmapStatus) { - return s.controlNetStatus.Load().(control.NetmapStatus) + return control.NetmapStatus(s.controlNetStatus.Load()) } func (s *networkState) getNodeInfo() (res netmapSDK.NodeInfo, ok bool) { diff --git a/go.mod b/go.mod index d90f45766..015ad5c2d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,6 @@ require ( go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.15.1 go.opentelemetry.io/otel/trace v1.15.1 - go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 @@ -102,6 +101,7 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.opentelemetry.io/otel/sdk v1.15.1 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index 493ae92de..1937f7a49 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -2,12 +2,12 @@ package innerring import ( "fmt" + "sync/atomic" "testing" "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestIndexerReturnsIndexes(t *testing.T) { @@ -209,7 +209,7 @@ type testCommiteeFetcher struct { } func (f *testCommiteeFetcher) Committee() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } @@ -220,6 +220,6 @@ type testIRFetcher struct { } func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 8c8c13dc3..17acdedda 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" @@ -29,7 +30,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -52,7 +52,7 @@ type ( epochDuration atomic.Uint64 statusIndex *innerRingIndexer precision precision.Fixed8Converter - healthStatus atomic.Value + healthStatus atomic.Int32 balanceClient *balanceClient.Client netmapClient *nmClient.Client persistate *state.PersistentStorage diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 6a6ca0ade..27f265ae2 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -153,7 +153,7 @@ func (s *Server) ResetEpochTimer(h uint32) error { } func (s *Server) setHealthStatus(hs control.HealthStatus) { - s.healthStatus.Store(hs) + s.healthStatus.Store(int32(hs)) if s.metrics != nil { s.metrics.SetHealth(int32(hs)) } @@ -161,7 +161,7 @@ func (s *Server) setHealthStatus(hs control.HealthStatus) { // HealthStatus returns the current health status of the IR application. func (s *Server) HealthStatus() control.HealthStatus { - return s.healthStatus.Load().(control.HealthStatus) + return control.HealthStatus(s.healthStatus.Load()) } func initPersistentStateStorage(cfg *viper.Viper) (*state.PersistentStorage, error) { diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index a49324406..21c9f9e58 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -3,11 +3,11 @@ package blobovnicza import ( "io/fs" "os" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 82454fa28..1cc100d19 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -44,7 +44,7 @@ func (b *Blobovnicza) incSize(sz uint64) { } func (b *Blobovnicza) decSize(sz uint64) { - b.filled.Sub(sz) + b.filled.Add(^(sz - 1)) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index d7eaae1d8..0c433f226 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "sync/atomic" "testing" "time" @@ -24,7 +25,6 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index b7be4756d..19e7b5237 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -3,6 +3,7 @@ package engine import ( "errors" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -10,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -131,7 +131,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) } @@ -149,7 +149,7 @@ func (e *StorageEngine) reportShardError( return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 4d2ddc100..9c239739f 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -22,7 +23,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -98,7 +98,7 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard te.engine.shards[s.ID().String()] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: s, }, hash: hrw.Hash([]byte(s.ID().String())), diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 4693b2618..d698fca74 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -17,7 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -41,9 +41,9 @@ type EvacuateShardRes struct { // NewEvacuateShardRes creates new EvacuateShardRes instance. func NewEvacuateShardRes() *EvacuateShardRes { return &EvacuateShardRes{ - evacuated: atomic.NewUint64(0), - total: atomic.NewUint64(0), - failed: atomic.NewUint64(0), + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), } } @@ -97,11 +97,17 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { if p == nil { return nil } - return &EvacuateShardRes{ - evacuated: atomic.NewUint64(p.evacuated.Load()), - total: atomic.NewUint64(p.total.Load()), - failed: atomic.NewUint64(p.failed.Load()), + + res := &EvacuateShardRes{ + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), } + + res.evacuated.Store(p.evacuated.Load()) + res.total.Store(p.total.Load()) + res.failed.Store(p.failed.Load()) + return res } const defaultEvacuateBatchSize = 100 @@ -323,7 +329,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { - res.failed.Inc() + res.failed.Add(1) continue } e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) @@ -350,7 +356,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - res.evacuated.Inc() + res.evacuated.Add(1) } return nil } @@ -371,7 +377,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { - res.evacuated.Inc() + res.evacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 64546d9ef..e16d2c498 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -2,6 +2,7 @@ package engine import ( "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -11,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -133,7 +133,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { e.shards[strID] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: sh, }, hash: hrw.Hash([]byte(strID)), diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 1a1f3cf9e..059f00b3a 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -2,13 +2,13 @@ package testutil import ( "encoding/binary" + "sync/atomic" "testing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "golang.org/x/exp/rand" ) @@ -27,7 +27,7 @@ var _ AddressGenerator = &SeqAddrGenerator{} func (g *SeqAddrGenerator) Next() oid.Address { var id oid.ID - binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.MaxID)+1) + binary.LittleEndian.PutUint64(id[:], ((g.cnt.Add(1)-1)%g.MaxID)+1) var addr oid.Address addr.SetContainer(cid.ID{}) addr.SetObject(id) @@ -69,7 +69,7 @@ func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *ob func (g *SeqObjGenerator) Next() *object.Object { var id oid.ID - binary.LittleEndian.PutUint64(id[:], g.cnt.Inc()) + binary.LittleEndian.PutUint64(id[:], g.cnt.Add(1)) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index a3a071d19..dbf89c9c8 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -4,6 +4,7 @@ import ( "context" "runtime" "strconv" + "sync/atomic" "testing" "time" @@ -15,7 +16,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func prepareObjects(t testing.TB, n int) []*objectSDK.Object { @@ -49,13 +49,15 @@ func BenchmarkPut(b *testing.B) { // Ensure the benchmark is bound by CPU and not waiting batch-delay time. b.SetParallelism(1) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) + objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } @@ -65,12 +67,13 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(1)) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 3f0a19cd4..82b107196 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -27,7 +28,6 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index e5ca85735..e8224ce5d 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "sync/atomic" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -22,7 +23,6 @@ import ( versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) @@ -132,7 +132,7 @@ func TestFlush(t *testing.T) { testIgnoreErrors := func(t *testing.T, f func(*cache)) { var errCount atomic.Uint32 wc, bs, mb := newCache(t, WithReportErrorFunc(func(message string, err error) { - errCount.Inc() + errCount.Add(1) })) objects := putObjects(t, wc) f(wc.(*cache)) diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 1ba5a4bd3..9c1c562b0 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -2,9 +2,10 @@ package writecache import ( "fmt" + "math" + "sync/atomic" "go.etcd.io/bbolt" - "go.uber.org/atomic" ) func (c *cache) estimateCacheSize() uint64 { @@ -24,11 +25,11 @@ type counters struct { } func (x *counters) IncDB() { - x.cDB.Inc() + x.cDB.Add(1) } func (x *counters) DecDB() { - x.cDB.Dec() + x.cDB.Add(math.MaxUint64) } func (x *counters) DB() uint64 { @@ -36,11 +37,11 @@ func (x *counters) DB() uint64 { } func (x *counters) IncFS() { - x.cFS.Inc() + x.cFS.Add(1) } func (x *counters) DecFS() { - x.cFS.Dec() + x.cFS.Add(math.MaxUint64) } func (x *counters) FS() uint64 { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index c9ff14a66..6b6e1284b 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -26,7 +27,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" - "go.uber.org/atomic" "go.uber.org/zap" ) diff --git a/pkg/util/worker_pool.go b/pkg/util/worker_pool.go index 145fd1a5a..97d76c492 100644 --- a/pkg/util/worker_pool.go +++ b/pkg/util/worker_pool.go @@ -1,8 +1,9 @@ package util import ( + "sync/atomic" + "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" ) // WorkerPool represents a tool to control -- 2.45.2 From cf8531ccd74ea79747f1ac23c4f852ed426a4e7f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 22 May 2023 11:30:03 +0300 Subject: [PATCH 049/233] [#381] go.mod: Update api-go and sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | Bin 97348 -> 97348 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 015ad5c2d..2c95bd41b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 648afba0e2cb76ed7094c6bf088c1dad3adda763..d728851b9b08097b80e81aa9be1e3b40cd9eaf61 100644 GIT binary patch delta 339 zcmZwBOHRT-007XiU`dt41F$qP=uDk{&=oj^mY=qD=+u=Awxw3uDy{vjOgsV?F>%2G z#Y~Auu9;yWA1AtPPgKjgSF_ko z--~ongvohd|2 bS_4NGSljPS;`~8?)G3gFhCmP)c53Spe37;eK0rQ{ z$JA2?_q~7i-R_#(UGw{ Date: Mon, 15 May 2023 15:14:56 +0300 Subject: [PATCH 050/233] [#338] adm: Drop notaryless code Signed-off-by: Dmitrii Stepanov --- .../internal/modules/morph/balance.go | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index f97250c38..6debc50b9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -37,11 +37,6 @@ const ( dumpBalancesAlphabetFlag = "alphabet" dumpBalancesProxyFlag = "proxy" dumpBalancesUseScriptHashFlag = "script-hash" - - // notaryEnabled signifies whether contracts were deployed in a notary-enabled environment. - // The setting is here to simplify testing and building the command for testnet (notary currently disabled). - // It will be removed eventually. - notaryEnabled = true ) func dumpBalances(cmd *cobra.Command, _ []string) error { @@ -60,7 +55,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) - if !notaryEnabled || dumpStorage || dumpAlphabet || dumpProxy { + if dumpStorage || dumpAlphabet || dumpProxy { nnsCs, err = c.GetContractStateByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) @@ -72,7 +67,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { } } - irList, err := fetchIRNodes(c, nmHash, rolemgmt.Hash) + irList, err := fetchIRNodes(c, rolemgmt.Hash) if err != nil { return err } @@ -187,40 +182,22 @@ func printAlphabetContractBalances(cmd *cobra.Command, c Client, inv *invoker.In return nil } -func fetchIRNodes(c Client, nmHash, desigHash util.Uint160) ([]accBalancePair, error) { - var irList []accBalancePair - +func fetchIRNodes(c Client, desigHash util.Uint160) ([]accBalancePair, error) { inv := invoker.New(c, nil) - if notaryEnabled { - height, err := c.GetBlockCount() - if err != nil { - return nil, fmt.Errorf("can't get block height: %w", err) - } + height, err := c.GetBlockCount() + if err != nil { + return nil, fmt.Errorf("can't get block height: %w", err) + } - arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } + arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) + if err != nil { + return nil, errors.New("can't fetch list of IR nodes from the netmap contract") + } - irList = make([]accBalancePair, len(arr)) - for i := range arr { - irList[i].scriptHash = arr[i].GetScriptHash() - } - } else { - arr, err := unwrap.ArrayOfBytes(inv.Call(nmHash, "innerRingList")) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } - - irList = make([]accBalancePair, len(arr)) - for i := range arr { - pub, err := keys.NewPublicKeyFromBytes(arr[i], elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("can't parse IR node public key: %w", err) - } - irList[i].scriptHash = pub.GetScriptHash() - } + irList := make([]accBalancePair, len(arr)) + for i := range arr { + irList[i].scriptHash = arr[i].GetScriptHash() } return irList, nil } -- 2.45.2 From fb708b3a2d5d3744a1b4b49e163e70a011b05f82 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 15:43:56 +0300 Subject: [PATCH 051/233] [#338] ir: Drop container notaryless code Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 2 +- .../processors/container/handlers_test.go | 84 +++++++++---------- .../processors/container/process_container.go | 40 +-------- .../processors/container/process_eacl.go | 18 +--- .../processors/container/processor.go | 43 +++++----- 5 files changed, 66 insertions(+), 121 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 89269d50d..6eb67ae7d 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -263,9 +263,9 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, + MorphClient: cnrClient.Morph(), FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, - NotaryDisabled: s.sideNotaryConfig.disabled, }) if err != nil { return err diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 6075ff577..1e518f474 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -7,7 +7,6 @@ import ( "time" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -22,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" "github.com/nspcc-dev/neo-go/pkg/util" @@ -34,18 +34,16 @@ func TestPutEvent(t *testing.T) { homHashDisabled: true, epoch: 100, } - cc := &testContainerClient{ - get: make(map[string]*containercore.Container), - } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, - ContainerClient: cc, + ContainerClient: &testContainerClient{}, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -64,10 +62,15 @@ func TestPutEvent(t *testing.T) { cnr.SetBasicACL(acl.Private) containerSDK.DisableHomomorphicHashing(&cnr) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + event := &testPutEvent{ cnr: &cnr, pk: p, st: nil, + nr: nr, } proc.handlePut(event) @@ -76,13 +79,7 @@ func TestPutEvent(t *testing.T) { time.Sleep(10 * time.Millisecond) } - var expectedPut cntClient.PutPrm - expectedPut.SetContainer(cnr.Marshal()) - expectedPut.SetKey(p.PublicKey().Bytes()) - expectedPut.SetSignature(p.Sign(cnr.Marshal())) - expectedPut.SetZone("container") - - require.EqualValues(t, []cntClient.PutPrm{expectedPut}, cc.put, "invalid put requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestDeleteEvent(t *testing.T) { @@ -103,15 +100,16 @@ func TestDeleteEvent(t *testing.T) { p.PublicKey(), }, } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: idc, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -133,9 +131,14 @@ func TestDeleteEvent(t *testing.T) { cidBin := make([]byte, 32) cid.Encode(cidBin) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + ev := containerEvent.Delete{ - ContainerIDValue: cidBin, - SignatureValue: p.Sign(cidBin), + ContainerIDValue: cidBin, + SignatureValue: p.Sign(cidBin), + NotaryRequestValue: nr, } var signature frostfscrypto.Signature @@ -156,7 +159,7 @@ func TestDeleteEvent(t *testing.T) { expectedDelete.SetCID(ev.ContainerID()) expectedDelete.SetSignature(ev.Signature()) - require.EqualValues(t, []cntClient.DeletePrm{expectedDelete}, cc.delete, "invalid delete requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestSetEACLEvent(t *testing.T) { @@ -168,15 +171,16 @@ func TestSetEACLEvent(t *testing.T) { cc := &testContainerClient{ get: make(map[string]*containercore.Container), } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -219,10 +223,14 @@ func TestSetEACLEvent(t *testing.T) { table.AddRecord(r) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } event := containerEvent.SetEACL{ - TableValue: table.ToV2().StableMarshal(nil), - PublicKeyValue: p.PublicKey().Bytes(), - SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + TableValue: table.ToV2().StableMarshal(nil), + PublicKeyValue: p.PublicKey().Bytes(), + SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + NotaryRequestValue: nr, } proc.handleSetEACL(event) @@ -236,7 +244,7 @@ func TestSetEACLEvent(t *testing.T) { expectedPutEACL.SetKey(p.PublicKey().Bytes()) expectedPutEACL.SetSignature(p.Sign(table.ToV2().StableMarshal(nil))) - require.EqualValues(t, []cntClient.PutEACLPrm{expectedPutEACL}, cc.putEACL, "invalid set EACL requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } type testAlphabetState struct { @@ -262,25 +270,13 @@ func (s *testNetworkState) Epoch() (uint64, error) { type testContainerClient struct { contractAddress util.Uint160 - put []cntClient.PutPrm get map[string]*containercore.Container - delete []cntClient.DeletePrm - putEACL []cntClient.PutEACLPrm } func (c *testContainerClient) ContractAddress() util.Uint160 { return c.contractAddress } -func (c *testContainerClient) Morph() *client.Client { - return nil -} - -func (c *testContainerClient) Put(p cntClient.PutPrm) error { - c.put = append(c.put, p) - return nil -} - func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) { key := hex.EncodeToString(cid) if cont, found := c.get[key]; found { @@ -289,16 +285,6 @@ func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) return nil, apistatus.ContainerNotFound{} } -func (c *testContainerClient) Delete(p cntClient.DeletePrm) error { - c.delete = append(c.delete, p) - return nil -} - -func (c *testContainerClient) PutEACL(p cntClient.PutEACLPrm) error { - c.putEACL = append(c.putEACL, p) - return nil -} - type testIDClient struct { publicKeys keys.PublicKeys } @@ -313,6 +299,7 @@ type testPutEvent struct { cnr *containerSDK.Container pk *keys.PrivateKey st []byte + nr *payload.P2PNotaryRequest } func (e *testPutEvent) MorphEvent() {} @@ -333,5 +320,14 @@ func (e *testPutEvent) SessionToken() []byte { return e.st } func (e *testPutEvent) NotaryRequest() *payload.P2PNotaryRequest { + return e.nr +} + +type testMorphClient struct { + transactions []*transaction.Transaction +} + +func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { + c.transactions = append(c.transactions, mainTx) return nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 603ab86d1..3bee1c4d5 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -4,7 +4,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -91,27 +90,7 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { } func (cp *Processor) approvePutContainer(ctx *putContainerContext) { - e := ctx.e - - var err error - - prm := cntClient.PutPrm{} - - prm.SetContainer(e.Container()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - prm.SetName(ctx.d.Name()) - prm.SetZone(ctx.d.Zone()) - - if nr := e.NotaryRequest(); nr != nil { - // put event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // put event was received via notification service - err = cp.cnrClient.Put(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApprovePutContainer, zap.String("error", err.Error()), ) @@ -171,22 +150,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { } func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { - var err error - - prm := cntClient.DeletePrm{} - - prm.SetCID(e.ContainerID()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // delete event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // delete event was received via notification service - err = cp.cnrClient.Delete(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index a8d880c8f..43a59e223 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -75,23 +75,7 @@ func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { } func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { - var err error - - prm := cntClient.PutEACLPrm{} - - prm.SetTable(e.Table()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // setEACL event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // setEACL event was received via notification service - err = cp.cnrClient.PutEACL(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApproveSetEACL, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 2141b0764..ec82ace70 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,13 +6,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" @@ -27,11 +26,11 @@ type ( ContClient interface { ContractAddress() util.Uint160 - Morph() *client.Client - Put(p cntClient.PutPrm) error Get(cid []byte) (*containercore.Container, error) - Delete(p cntClient.DeletePrm) error - PutEACL(p cntClient.PutEACLPrm) error + } + + MorphClient interface { + NotarySignAndInvokeTX(mainTx *transaction.Transaction) error } IDClient interface { @@ -40,13 +39,13 @@ type ( // Processor of events produced by container contract in the sidechain. Processor struct { - log *logger.Logger - pool *ants.Pool - alphabetState AlphabetState - cnrClient ContClient // notary must be enabled - idClient IDClient - netState NetworkState - notaryDisabled bool + log *logger.Logger + pool *ants.Pool + alphabetState AlphabetState + cnrClient ContClient // notary must be enabled + morphClient MorphClient + idClient IDClient + netState NetworkState } // Params of the processor constructor. @@ -55,9 +54,9 @@ type ( PoolSize int AlphabetState AlphabetState ContainerClient ContClient + MorphClient MorphClient FrostFSIDClient IDClient NetworkState NetworkState - NotaryDisabled bool } ) @@ -88,6 +87,8 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: global state is not set") case p.ContainerClient == nil: return nil, errors.New("ir/container: Container client is not set") + case p.MorphClient == nil: + return nil, errors.New("ir/container: Morph client is not set") case p.FrostFSIDClient == nil: return nil, errors.New("ir/container: FrostFS ID client is not set") case p.NetworkState == nil: @@ -102,13 +103,13 @@ func New(p *Params) (*Processor, error) { } return &Processor{ - log: p.Log, - pool: pool, - alphabetState: p.AlphabetState, - cnrClient: p.ContainerClient, - idClient: p.FrostFSIDClient, - netState: p.NetworkState, - notaryDisabled: p.NotaryDisabled, + log: p.Log, + pool: pool, + alphabetState: p.AlphabetState, + cnrClient: p.ContainerClient, + idClient: p.FrostFSIDClient, + netState: p.NetworkState, + morphClient: p.MorphClient, }, nil } -- 2.45.2 From 656fd7f376db7ad97f33184ca1b4cb576303a23b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 16:56:47 +0300 Subject: [PATCH 052/233] [#338] ir: Drop notaryless code from netmap Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 1 - .../processors/netmap/handlers_test.go | 361 ++++++------------ .../processors/netmap/process_cleanup.go | 26 +- .../processors/netmap/process_peers.go | 61 +-- pkg/innerring/processors/netmap/processor.go | 9 - 5 files changed, 136 insertions(+), 322 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 6eb67ae7d..50ba67124 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -71,7 +71,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, addrvalidator.New(), locodeValidator, ), - NotaryDisabled: s.sideNotaryConfig.disabled, NodeStateSettings: netSettings, }) diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 4905b45d2..6c9e265cc 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -31,7 +31,6 @@ func TestNewEpochTick(t *testing.T) { nc := &testNetmapClient{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.CleanupEnabled = true p.EpochState = es p.NetmapClient = nc @@ -80,7 +79,6 @@ func TestNewEpoch(t *testing.T) { eh := &testEventHandler{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.NotaryDepositHandler = eh.Handle p.AlphabetSyncHandler = eh.Handle p.NetmapClient = nc @@ -119,276 +117,139 @@ func TestNewEpoch(t *testing.T) { func TestAddPeer(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + nc := &testNetmapClient{ + contractAddress: util.Uint160{47}, + } - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key1") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: ev.Request.MainTransaction.Nonce, - vub: nil, - method: "addPeerIR", - args: []any{ev.Node()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() + require.NoError(t, err, "failed to create processor") - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + var node netmap.NodeInfo + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + node.SetPublicKey(key.Bytes()) - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) + ev := netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleAddPeer(ev) - require.NoError(t, err, "failed to create processor") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var addPeerExp netmapclient.AddPeerPrm - addPeerExp.SetNodeInfo(node) - require.EqualValues(t, []netmapclient.AddPeerPrm{addPeerExp}, nc.addPeers, "invalid peers") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: ev.NotaryRequest().MainTransaction.Nonce, + vub: nil, + method: "addPeerIR", + args: []any{node.Marshal()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") } func TestUpdateState(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + ns := &testNodeStateSettings{ + maintAllowed: true, + } + nc := &testNetmapClient{} - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NodeStateSettings = ns - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []*transaction.Transaction{ - ev.Request.MainTransaction, - }, nc.invokedTxs, "invalid invoked transactions") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc + p.NodeStateSettings = ns }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + require.NoError(t, err, "failed to create processor") - proc, err := newTestProc(t, func(p *Params) { - p.NetmapClient = nc - p.NodeStateSettings = ns - }) + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") - require.NoError(t, err, "failed to create processor") + ev := netmapEvent.UpdatePeer{ + State: netmapContract.NodeStateOnline, + PubKey: key, + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleUpdateState(ev) - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var expUpdPeer netmapclient.UpdatePeerPrm - expUpdPeer.SetMaintenance() - expUpdPeer.SetOnline() - expUpdPeer.SetKey(ev.PubKey.Bytes()) - - require.EqualValues(t, []netmapclient.UpdatePeerPrm{expUpdPeer}, nc.peerStateUpdates, "invalid peer state updates") - }) + require.EqualValues(t, []*transaction.Transaction{ev.Request.MainTransaction}, nc.invokedTxs, "invalid transactions") } func TestCleanupTick(t *testing.T) { t.Parallel() - t.Run("notary disabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{} - - proc, err := newTestProc(t, func(p *Params) { + nc := &testNetmapClient{ + contractAddress: util.Uint160{111}, + } + proc, err := newTestProc(t, + func(p *Params) { p.NetmapClient = nc - p.NotaryDisabled = true p.CleanupEnabled = true - }) + }, + ) - require.NoError(t, err, "failed to create processor") + require.NoError(t, err, "failed to create processor") - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } + key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" + proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 95, + removeFlag: false, + }, + } + key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" + proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 98, + removeFlag: false, + }, + } - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } + ev := netmapCleanupTick{ + epoch: 100, + txHash: util.Uint256{123}, + } - proc.handleCleanupTick(ev) + proc.handleCleanupTick(ev) - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") + keyExp, err := keys.NewPublicKeyFromString(key1Str) + require.NoError(t, err, "failed to parse expired key") - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) + updExp := netmapclient.UpdatePeerPrm{} + updExp.SetKey(keyExp.Bytes()) + updExp.SetHash(ev.TxHash()) - require.EqualValues(t, []netmapclient.UpdatePeerPrm{updExp}, nc.peerStateUpdates, "invalid peer updates") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) - - t.Run("notary enabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{ - contractAddress: util.Uint160{111}, - } - proc, err := newTestProc(t, - func(p *Params) { - p.NetmapClient = nc - p.CleanupEnabled = true - }, - ) - - require.NoError(t, err, "failed to create processor") - - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } - - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } - - proc.handleCleanupTick(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") - - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: uint32(ev.epoch), - vub: nil, - method: "updateStateIR", - args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: uint32(ev.epoch), + vub: nil, + method: "updateStateIR", + args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") + require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") + require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") } func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { @@ -407,7 +268,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { PoolSize: 1, CleanupEnabled: false, CleanupThreshold: 3, - NotaryDisabled: false, NodeStateSettings: ns, NodeValidator: &testValidator{}, EpochState: es, @@ -500,17 +360,11 @@ type testNetmapClient struct { netmap *netmap.NetMap txHeights map[util.Uint256]uint32 - peerStateUpdates []netmapclient.UpdatePeerPrm - notaryInvokes []notaryInvoke - newEpochs []uint64 - addPeers []netmapclient.AddPeerPrm - invokedTxs []*transaction.Transaction + notaryInvokes []notaryInvoke + newEpochs []uint64 + invokedTxs []*transaction.Transaction } -func (c *testNetmapClient) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { - c.peerStateUpdates = append(c.peerStateUpdates, p) - return nil -} func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { c.notaryInvokes = append(c.notaryInvokes, notaryInvoke{ contract: contract, @@ -522,32 +376,35 @@ func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.F }) return nil } + func (c *testNetmapClient) ContractAddress() util.Uint160 { return c.contractAddress } + func (c *testNetmapClient) EpochDuration() (uint64, error) { return c.epochDuration, nil } + func (c *testNetmapClient) MorphTxHeight(h util.Uint256) (uint32, error) { if res, found := c.txHeights[h]; found { return res, nil } return 0, fmt.Errorf("not found") } + func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { return c.netmap, nil } + func (c *testNetmapClient) NewEpoch(epoch uint64, force bool) error { c.newEpochs = append(c.newEpochs, epoch) return nil } + func (c *testNetmapClient) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { return true, nil } -func (c *testNetmapClient) AddPeer(p netmapclient.AddPeerPrm) error { - c.addPeers = append(c.addPeers, p) - return nil -} + func (c *testNetmapClient) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { c.invokedTxs = append(c.invokedTxs, mainTx) return nil diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 45a08b377..287844a6a 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -3,7 +3,6 @@ package netmap import ( v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) @@ -31,23 +30,14 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { // See https://github.com/nspcc-dev/frostfs-contract/issues/225 const methodUpdateStateNotary = "updateStateIR" - if np.notaryDisabled { - prm := netmapclient.UpdatePeerPrm{} - - prm.SetKey(key.Bytes()) - prm.SetHash(ev.TxHash()) - - err = np.netmapClient.UpdatePeerState(prm) - } else { - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - uint32(ev.epoch), - nil, - methodUpdateStateNotary, - int64(v2netmap.Offline), key.Bytes(), - ) - } + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + uint32(ev.epoch), + nil, + methodUpdateStateNotary, + int64(v2netmap.Offline), key.Bytes(), + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 9e6eeb53e..e4f1a4d6c 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -19,16 +19,14 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { } // check if notary transaction is valid, see #976 - if originalRequest := ev.NotaryRequest(); originalRequest != nil { - tx := originalRequest.MainTransaction - ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) - if err != nil || !ok { - np.log.Warn(logs.NetmapNonhaltNotaryTransaction, - zap.String("method", "netmap.AddPeer"), - zap.String("hash", tx.Hash().StringLE()), - zap.Error(err)) - return - } + tx := ev.NotaryRequest().MainTransaction + ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) + if err != nil || !ok { + np.log.Warn(logs.NetmapNonhaltNotaryTransaction, + zap.String("method", "netmap.AddPeer"), + zap.String("hash", tx.Hash().StringLE()), + zap.Error(err)) + return } // unmarshal node info @@ -40,7 +38,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { } // validate and update node info - err := np.nodeValidator.VerifyAndUpdate(&nodeInfo) + err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { np.log.Warn(logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), @@ -71,20 +69,15 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { // See https://github.com/nspcc-dev/frostfs-contract/issues/154. const methodAddPeerNotary = "addPeerIR" - if nr := ev.NotaryRequest(); nr != nil { - // create new notary request with the original nonce - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - nr.MainTransaction.Nonce, - nil, - methodAddPeerNotary, - nodeInfoBinary, - ) - } else { - // notification event case - err = np.netmapClient.AddPeer(prm) - } + // create new notary request with the original nonce + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + ev.NotaryRequest().MainTransaction.Nonce, + nil, + methodAddPeerNotary, + nodeInfoBinary, + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) @@ -116,23 +109,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { } } - if nr := ev.NotaryRequest(); nr != nil { - err = np.netmapClient.MorphNotarySignAndInvokeTX(nr.MainTransaction) - } else { - prm := netmapclient.UpdatePeerPrm{} - - switch { - case ev.Online(): - prm.SetOnline() - case ev.Maintenance(): - prm.SetMaintenance() - } - - prm.SetKey(ev.PublicKey().Bytes()) - - err = np.netmapClient.UpdatePeerState(prm) - } - if err != nil { + if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) } } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index c466cfb1b..5984cbbe9 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -55,7 +54,6 @@ type ( } Client interface { - UpdatePeerState(p netmapclient.UpdatePeerPrm) error MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error ContractAddress() util.Uint160 EpochDuration() (uint64, error) @@ -63,7 +61,6 @@ type ( NetMap() (*netmap.NetMap, error) NewEpoch(epoch uint64, force bool) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) - AddPeer(p netmapclient.AddPeerPrm) error MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } @@ -90,8 +87,6 @@ type ( nodeValidator NodeValidator - notaryDisabled bool - nodeStateSettings state.NetworkSettings } @@ -112,8 +107,6 @@ type ( NodeValidator NodeValidator - NotaryDisabled bool - NodeStateSettings state.NetworkSettings } ) @@ -168,8 +161,6 @@ func New(p *Params) (*Processor, error) { nodeValidator: p.NodeValidator, - notaryDisabled: p.NotaryDisabled, - nodeStateSettings: p.NodeStateSettings, }, nil } -- 2.45.2 From 81718afb397c8598bb43eb28d0d4cd7e9522b03a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 17:41:28 +0300 Subject: [PATCH 053/233] [#338] ir: Drop named named put fee Named put fee value used only when notary disabled. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/defaults.go | 5 ++-- config/example/ir.env | 1 - config/example/ir.yaml | 1 - pkg/innerring/config/fee.go | 13 ++--------- pkg/innerring/config/fee_test.go | 4 ---- pkg/innerring/initialization.go | 8 ------- pkg/morph/client/container/client.go | 15 ------------ pkg/morph/client/fee.go | 35 ---------------------------- pkg/morph/client/fee_test.go | 32 ------------------------- pkg/morph/client/static.go | 20 ++++------------ 10 files changed, 9 insertions(+), 125 deletions(-) delete mode 100644 pkg/morph/client/fee.go delete mode 100644 pkg/morph/client/fee_test.go diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 007bb8964..a7fe8d563 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -51,9 +51,8 @@ func setControlDefaults(cfg *viper.Viper) { func setFeeDefaults(cfg *viper.Viper) { // extra fee values for working mode without notary contract - cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 - cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 - cfg.SetDefault("fee.named_container_register", 25_0000_0000) // 25.0 Fixed8 + cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 + cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 } func setEmitDefaults(cfg *viper.Viper) { diff --git a/config/example/ir.env b/config/example/ir.env index 7b8f8a89d..3f9530ab6 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -28,7 +28,6 @@ FROSTFS_IR_LOCODE_DB_PATH=/path/to/locode.db FROSTFS_IR_FEE_MAIN_CHAIN=50000000 FROSTFS_IR_FEE_SIDE_CHAIN=200000000 -FROSTFS_IR_FEE_NAMED_CONTAINER_REGISTER=2500000000 FROSTFS_IR_TIMERS_EMIT=240 FROSTFS_IR_TIMERS_STOP_ESTIMATION_MUL=1 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 1130a840b..a01f3d0bb 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -49,7 +49,6 @@ locode: fee: main_chain: 50000000 # Fixed8 value of extra GAS fee for mainchain contract invocation; ignore if notary is enabled in mainchain side_chain: 200000000 # Fixed8 value of extra GAS fee for sidechain contract invocation; ignore if notary is enabled in sidechain - named_container_register: 2500000000 # Fixed8 value of extra GAS fee for named conatiner registration in container contract; ignore if notary is enabled in sidechain timers: emit: 240 # Number of sidechain blocks between GAS emission cycles; disabled by default diff --git a/pkg/innerring/config/fee.go b/pkg/innerring/config/fee.go index d77685643..a26a7bcc6 100644 --- a/pkg/innerring/config/fee.go +++ b/pkg/innerring/config/fee.go @@ -8,19 +8,15 @@ import ( // FeeConfig is an instance that returns extra fee values for contract // invocations without notary support. type FeeConfig struct { - registerNamedCnr, mainchain, sidechain fixedn.Fixed8 } // NewFeeConfig constructs FeeConfig from viper.Viper instance. Latter must not be nil. -// -// Fee for named container registration is taken from "fee.named_container_register" value. func NewFeeConfig(v *viper.Viper) *FeeConfig { return &FeeConfig{ - registerNamedCnr: fixedn.Fixed8(v.GetInt64("fee.named_container_register")), - mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), - sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), + mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), + sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), } } @@ -31,8 +27,3 @@ func (f FeeConfig) MainChainFee() fixedn.Fixed8 { func (f FeeConfig) SideChainFee() fixedn.Fixed8 { return f.sidechain } - -// NamedContainerRegistrationFee returns additional GAS fee for named container registration in FrostFS network. -func (f FeeConfig) NamedContainerRegistrationFee() fixedn.Fixed8 { - return f.registerNamedCnr -} diff --git a/pkg/innerring/config/fee_test.go b/pkg/innerring/config/fee_test.go index a0c56aac1..f7330c6ca 100644 --- a/pkg/innerring/config/fee_test.go +++ b/pkg/innerring/config/fee_test.go @@ -18,7 +18,6 @@ func TestConfig(t *testing.T) { fee: main_chain: 50000000 side_chain: 200000000 - named_container_register: 2500000000 `, ) v := viper.New() @@ -29,7 +28,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(50000000), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(200000000), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(2500000000), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("nothing set", func(t *testing.T) { @@ -43,7 +41,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(0), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("partially set", func(t *testing.T) { @@ -62,7 +59,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(10), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 50ba67124..9b5634382 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -387,14 +387,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { container.AsAlphabet(), ) - if s.sideNotaryConfig.disabled { - // in non-notary environments we customize fee for named container registration - // because it takes much more additional GAS than other operations. - morphCnrOpts = append(morphCnrOpts, - container.WithCustomFeeForNamedPut(s.feeConfig.NamedContainerRegistrationFee()), - ) - } - result.CnrClient, err = container.NewFromMorph(s.morphClient, s.contracts.container, fee, morphCnrOpts...) if err != nil { return nil, err diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 85d742328..2b5996cd7 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -56,10 +56,6 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts[i](o) } - if o.feePutNamedSet { - o.staticOpts = append(o.staticOpts, client.WithCustomFee(putNamedMethod, o.feePutNamed)) - } - sc, err := client.NewStatic(cli, contract, fee, o.staticOpts...) if err != nil { return nil, fmt.Errorf("can't create container static client: %w", err) @@ -83,9 +79,6 @@ func (c Client) ContractAddress() util.Uint160 { type Option func(*opts) type opts struct { - feePutNamedSet bool - feePutNamed fixedn.Fixed8 - staticOpts []client.StaticClientOption } @@ -111,11 +104,3 @@ func AsAlphabet() Option { o.staticOpts = append(o.staticOpts, client.AsAlphabet()) } } - -// WithCustomFeeForNamedPut returns option to specify custom fee for each Put operation with named container. -func WithCustomFeeForNamedPut(fee fixedn.Fixed8) Option { - return func(o *opts) { - o.feePutNamed = fee - o.feePutNamedSet = true - } -} diff --git a/pkg/morph/client/fee.go b/pkg/morph/client/fee.go deleted file mode 100644 index 8a38c4f55..000000000 --- a/pkg/morph/client/fee.go +++ /dev/null @@ -1,35 +0,0 @@ -package client - -import "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - -// setFeeForMethod sets fee for the operation executed using specified contract method. -func (x *fees) setFeeForMethod(method string, fee fixedn.Fixed8) { - if x.customFees == nil { - x.customFees = make(map[string]fixedn.Fixed8, 1) - } - - x.customFees[method] = fee -} - -// fees represents source of per-operation fees. -// Can be initialized using var declaration. -// -// Instances are not thread-safe, so they mean initially filling, and then only reading. -type fees struct { - defaultFee fixedn.Fixed8 - - // customFees represents source of customized per-operation fees. - customFees map[string]fixedn.Fixed8 -} - -// returns fee for the operation executed using specified contract method. -// Returns customized value if it is set. Otherwise, returns default value. -func (x fees) feeForMethod(method string) fixedn.Fixed8 { - if x.customFees != nil { - if fee, ok := x.customFees[method]; ok { - return fee - } - } - - return x.defaultFee -} diff --git a/pkg/morph/client/fee_test.go b/pkg/morph/client/fee_test.go deleted file mode 100644 index 963d64ce4..000000000 --- a/pkg/morph/client/fee_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package client - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/stretchr/testify/require" -) - -func TestFees(t *testing.T) { - var v fees - - const method = "some method" - - var ( - fee fixedn.Fixed8 - def = fixedn.Fixed8(13) - ) - - v.defaultFee = def - - fee = v.feeForMethod(method) - require.True(t, fee.Equal(def)) - - const customFee = fixedn.Fixed8(10) - - v.setFeeForMethod(method, customFee) - - fee = v.feeForMethod(method) - - require.Equal(t, customFee, fee) -} diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 910f78537..7aa17a70f 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -27,7 +27,7 @@ type staticOpts struct { tryNotary bool alpha bool // use client's key to sign notary request's main TX - fees fees + fee fixedn.Fixed8 } // WithNotary returns notary status of the client. @@ -63,7 +63,7 @@ func NewStatic(client *Client, scriptHash util.Uint160, fee fixedn.Fixed8, opts scScriptHash: scriptHash, } - c.fees.defaultFee = fee + c.fee = fee for i := range opts { opts[i](&c.staticOpts) @@ -125,8 +125,6 @@ func (i *InvokePrmOptional) SetControlTX(b bool) { // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. func (s StaticClient) Invoke(prm InvokePrm) error { - fee := s.fees.feeForMethod(prm.method) - if s.tryNotary { if s.alpha { var ( @@ -149,15 +147,15 @@ func (s StaticClient) Invoke(prm InvokePrm) error { vubP = &vub } - return s.client.NotaryInvoke(s.scScriptHash, fee, nonce, vubP, prm.method, prm.args...) + return s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) } - return s.client.NotaryInvokeNotAlpha(s.scScriptHash, fee, prm.method, prm.args...) + return s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) } return s.client.Invoke( s.scScriptHash, - fee, + s.fee, prm.method, prm.args..., ) @@ -211,11 +209,3 @@ func AsAlphabet() StaticClientOption { o.alpha = true } } - -// WithCustomFee returns option to specify custom fee for the operation executed using -// specified contract method. -func WithCustomFee(method string, fee fixedn.Fixed8) StaticClientOption { - return func(o *staticOpts) { - o.fees.setFeeForMethod(method, fee) - } -} -- 2.45.2 From 0ab589dd52e80f0fd3ab562b77b82761e1527397 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 17:56:11 +0300 Subject: [PATCH 054/233] [#338] ir: Always enable notary on sidechain Signed-off-by: Dmitrii Stepanov --- pkg/innerring/contracts.go | 10 ++++------ pkg/innerring/contracts_test.go | 18 ++++++++++-------- pkg/innerring/initialization.go | 24 ++++++++++-------------- pkg/innerring/innerring.go | 14 +++++--------- pkg/innerring/notary.go | 17 +++-------------- 5 files changed, 32 insertions(+), 51 deletions(-) diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index 55c2ff582..4a80296f4 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -22,7 +22,7 @@ type contracts struct { alphabet AlphabetContracts // in morph } -func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { +func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary bool) (*contracts, error) { var ( result = new(contracts) err error @@ -42,11 +42,9 @@ func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, without } } - if !withoutSideNotary { - result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) - if err != nil { - return nil, fmt.Errorf("can't get proxy script hash: %w", err) - } + result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) + if err != nil { + return nil, fmt.Errorf("can't get proxy script hash: %w", err) } targets := [...]struct { diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go index e04e22f06..5f877133c 100644 --- a/pkg/innerring/contracts_test.go +++ b/pkg/innerring/contracts_test.go @@ -35,7 +35,7 @@ contracts: t.Run("all enabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, false, false) + c, err := parseContracts(v, nil, false, false) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -70,7 +70,7 @@ contracts: t.Run("all disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, true, true, true) + c, err := parseContracts(v, nil, true, true) require.NoError(t, err, "failed to parse contracts") require.Equal(t, util.Uint160{}, c.frostfs, "invalid frostfs") @@ -89,7 +89,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -100,9 +101,9 @@ contracts: require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") }) - t.Run("main notary & side notary disabled", func(t *testing.T) { + t.Run("main notary disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, true, true) + c, err := parseContracts(v, nil, false, true) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -122,7 +123,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -159,7 +161,7 @@ contracts: err := v.ReadConfig(file) require.NoError(t, err, "read config file failed") - _, err = parseContracts(v, nil, false, false, false) + _, err = parseContracts(v, nil, false, false) require.Error(t, err, "unexpected success") }) @@ -196,7 +198,7 @@ contracts: }, } - _, err = parseContracts(v, morph, false, false, false) + _, err = parseContracts(v, morph, false, false) require.ErrorContains(t, err, "could not read all contracts: required 3, read 2", "unexpected success") }) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 9b5634382..076d7be43 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -116,18 +116,16 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * } func (s *Server) enableNotarySupport() error { - if !s.sideNotaryConfig.disabled { - // enable notary support in the side client - err := s.morphClient.EnableNotarySupport( - client.WithProxyContract(s.contracts.proxy), - ) - if err != nil { - return fmt.Errorf("could not enable side chain notary support: %w", err) - } - - s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + // enable notary support in the side client + err := s.morphClient.EnableNotarySupport( + client.WithProxyContract(s.contracts.proxy), + ) + if err != nil { + return fmt.Errorf("could not enable side chain notary support: %w", err) } + s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + if !s.mainNotaryConfig.disabled { // enable notary support in the main client err := s.mainnetClient.EnableNotarySupport( @@ -143,13 +141,12 @@ func (s *Server) enableNotarySupport() error { } func (s *Server) initNotaryConfig() { - s.mainNotaryConfig, s.sideNotaryConfig = notaryConfigs( - s.morphClient.ProbeNotary(), + s.mainNotaryConfig = notaryConfigs( !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) s.log.Info(logs.InnerringNotarySupport, - zap.Bool("sidechain_enabled", !s.sideNotaryConfig.disabled), + zap.Bool("sidechain_enabled", true), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) } @@ -500,7 +497,6 @@ func (s *Server) initContracts(cfg *viper.Viper) error { s.morphClient, s.withoutMainNet, s.mainNotaryConfig.disabled, - s.sideNotaryConfig.disabled, ) return err diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 17acdedda..3a690bbbe 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -63,7 +63,6 @@ type ( // notary configuration feeConfig *config.FeeConfig mainNotaryConfig *notaryConfig - sideNotaryConfig *notaryConfig // internal variables key *keys.PrivateKey @@ -267,14 +266,11 @@ func (s *Server) initMainNotary(ctx context.Context) error { } func (s *Server) initSideNotary(ctx context.Context) error { - if !s.sideNotaryConfig.disabled { - return s.initNotary(ctx, - s.depositSideNotary, - s.awaitSideNotaryDeposit, - "waiting to accept side notary deposit", - ) - } - return nil + return s.initNotary(ctx, + s.depositSideNotary, + s.awaitSideNotaryDeposit, + "waiting to accept side notary deposit", + ) } func (s *Server) tickInitialExpoch() { diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 30916cb99..c601f5587 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -57,11 +57,8 @@ func (s *Server) notaryHandler(_ event.Event) { } } - if !s.sideNotaryConfig.disabled { - _, err := s.depositSideNotary() - if err != nil { - s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) - } + if _, err := s.depositSideNotary(); err != nil { + s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } @@ -115,16 +112,8 @@ func awaitNotaryDepositInClient(ctx context.Context, cli *client.Client, txHash return errDepositTimeout } -func notaryConfigs(withSideNotary, withMainNotary bool) (main, side *notaryConfig) { +func notaryConfigs(withMainNotary bool) (main *notaryConfig) { main = new(notaryConfig) - side = new(notaryConfig) - - if !withSideNotary { - main.disabled = true - side.disabled = true - - return - } main.disabled = !withMainNotary -- 2.45.2 From 53a1b15693e3a14ec6343c2ef02dd080c6b752d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 11:19:08 +0300 Subject: [PATCH 055/233] [#338] ir: Drop notaryless code from governance Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 19 +++---- .../processors/governance/handlers_test.go | 56 +++++++++---------- .../processors/governance/process_update.go | 25 ++------- .../processors/governance/processor.go | 27 ++++----- 4 files changed, 52 insertions(+), 75 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 076d7be43..8e5aef951 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -161,16 +161,15 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli } else { // create governance processor governanceProcessor, err := governance.New(&governance.Params{ - Log: s.log, - FrostFSClient: frostfsCli, - NetmapClient: s.netmapClient, - AlphabetState: s, - EpochState: s, - Voter: s, - IRFetcher: irf, - MorphClient: s.morphClient, - MainnetClient: s.mainnetClient, - NotaryDisabled: s.sideNotaryConfig.disabled, + Log: s.log, + FrostFSClient: frostfsCli, + NetmapClient: s.netmapClient, + AlphabetState: s, + EpochState: s, + Voter: s, + IRFetcher: irf, + MorphClient: s.morphClient, + MainnetClient: s.mainnetClient, }) if err != nil { return nil, err diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 23f91869a..63d156dac 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -42,16 +42,15 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: true, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) @@ -74,17 +73,19 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { }, }, v.votes, "invalid vote calls") - var irUpdateExp nmClient.UpdateIRPrm - irUpdateExp.SetKeys(testKeys.newInnerRingExp) - irUpdateExp.SetHash(ev.txHash) + var irUpdateExp []nmClient.UpdateIRPrm - require.EqualValues(t, []nmClient.UpdateIRPrm{irUpdateExp}, nm.updates, "invalid IR updates") + require.EqualValues(t, irUpdateExp, nm.updates, "invalid IR updates") - var expAlphabetUpdates []client.UpdateAlphabetListPrm - require.EqualValues(t, expAlphabetUpdates, m.alphabetUpdates, "invalid alphabet updates") + var expAlphabetUpdate client.UpdateAlphabetListPrm + expAlphabetUpdate.SetHash(ev.txHash) + expAlphabetUpdate.SetList(testKeys.newInnerRingExp) + require.EqualValues(t, []client.UpdateAlphabetListPrm{expAlphabetUpdate}, m.alphabetUpdates, "invalid alphabet updates") - var expNotaryUpdates []client.UpdateNotaryListPrm - require.EqualValues(t, expNotaryUpdates, m.notaryUpdates, "invalid notary list updates") + var expNotaryUpdate client.UpdateNotaryListPrm + expNotaryUpdate.SetHash(ev.txHash) + expNotaryUpdate.SetList(testKeys.newAlphabetExp) + require.EqualValues(t, []client.UpdateNotaryListPrm{expNotaryUpdate}, m.notaryUpdates, "invalid notary list updates") buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, es.epoch) @@ -122,16 +123,15 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: false, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 629d8741e..3eae676d4 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" @@ -114,33 +113,17 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl zap.String("after", prettyKeys(newInnerRing)), ) - if gp.notaryDisabled { - updPrm := nmClient.UpdateIRPrm{} + updPrm := client.UpdateAlphabetListPrm{} + updPrm.SetList(newInnerRing) + updPrm.SetHash(txHash) - updPrm.SetKeys(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.netmapClient.UpdateInnerRing(updPrm) - } else { - updPrm := client.UpdateAlphabetListPrm{} - - updPrm.SetList(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm) - } - - if err != nil { + if err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm); err != nil { gp.log.Error(logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, txHash util.Uint256) { - if gp.notaryDisabled { - return - } - updPrm := client.UpdateNotaryListPrm{} updPrm.SetList(newAlphabet) diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index e08bd3809..07b5b5cef 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -87,8 +87,6 @@ type ( mainnetClient MainnetClient morphClient MorphClient - notaryDisabled bool - designate util.Uint160 } @@ -105,8 +103,6 @@ type ( MainnetClient MainnetClient FrostFSClient FrostFSClient NetmapClient NetmapClient - - NotaryDisabled bool } ) @@ -138,18 +134,17 @@ func New(p *Params) (*Processor, error) { designate := p.MainnetClient.GetDesignateHash() return &Processor{ - log: p.Log, - pool: pool, - frostfsClient: p.FrostFSClient, - netmapClient: p.NetmapClient, - alphabetState: p.AlphabetState, - epochState: p.EpochState, - voter: p.Voter, - irFetcher: p.IRFetcher, - mainnetClient: p.MainnetClient, - morphClient: p.MorphClient, - notaryDisabled: p.NotaryDisabled, - designate: designate, + log: p.Log, + pool: pool, + frostfsClient: p.FrostFSClient, + netmapClient: p.NetmapClient, + alphabetState: p.AlphabetState, + epochState: p.EpochState, + voter: p.Voter, + irFetcher: p.IRFetcher, + mainnetClient: p.MainnetClient, + morphClient: p.MorphClient, + designate: designate, }, nil } -- 2.45.2 From 4503a61997aadaefa4bb3988b5d3cf42b53d95ff Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 15:38:41 +0300 Subject: [PATCH 056/233] [#312] wc: Delete unused Iterate method Signed-off-by: Dmitrii Stepanov --- .../writecache/iterate.go | 54 ------------------- .../writecache/writecache.go | 1 - 2 files changed, 55 deletions(-) diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index ebe979520..59ace93bd 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -12,59 +11,6 @@ import ( // ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing. var ErrNoDefaultBucket = errors.New("no default bucket") -// IterationPrm contains iteration parameters. -type IterationPrm struct { - handler func([]byte) error - ignoreErrors bool -} - -// WithHandler sets a callback to be executed on every object. -func (p *IterationPrm) WithHandler(f func([]byte) error) { - p.handler = f -} - -// WithIgnoreErrors sets a flag indicating that errors should be ignored. -func (p *IterationPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// Iterate iterates over all objects present in write cache. -// This is very difficult to do correctly unless write-cache is put in read-only mode. -// Thus we silently fail if shard is not in read-only mode to avoid reporting misleading results. -func (c *cache) Iterate(prm IterationPrm) error { - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - if !c.readOnly() { - return nil - } - - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.ForEach(func(k, data []byte) error { - return prm.handler(data) - }) - }) - if err != nil { - return err - } - - var fsPrm common.IteratePrm - fsPrm.IgnoreErrors = prm.ignoreErrors - fsPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - data, err := f() - if err != nil { - if prm.ignoreErrors { - return nil - } - return err - } - return prm.handler(data) - } - - _, err = c.fsTree.Iterate(fsPrm) - return err -} - // IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return. // It is assumed that db is an underlying database of some WriteCache instance. // diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 83ecf219c..0edf4b9be 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -32,7 +32,6 @@ type Cache interface { // Returns apistatus.ObjectNotFound if object is missing in the Cache. // Returns ErrReadOnly if the Cache is currently in the read-only mode. Delete(context.Context, oid.Address) error - Iterate(IterationPrm) error Put(context.Context, common.PutPrm) (common.PutRes, error) SetMode(mode.Mode) error SetLogger(*logger.Logger) -- 2.45.2 From d212d908b5303b56e8b7eabe82560b4a8f58cf4c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 17:19:41 +0300 Subject: [PATCH 057/233] [#312] wc: Add metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/delete.go | 19 +++++++-- pkg/local_object_storage/writecache/flush.go | 14 +++++-- pkg/local_object_storage/writecache/get.go | 22 +++++++++- .../writecache/metrics.go | 42 +++++++++++++++++++ pkg/local_object_storage/writecache/mode.go | 6 ++- .../writecache/options.go | 9 ++++ pkg/local_object_storage/writecache/put.go | 23 +++++++++- pkg/local_object_storage/writecache/state.go | 5 ++- .../writecache/storage.go | 2 + .../writecache/writecache.go | 2 + 10 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 pkg/local_object_storage/writecache/metrics.go diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index c1aab9e5a..ed44a8ad8 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -2,6 +2,7 @@ package writecache import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -22,6 +23,13 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { )) defer span.End() + deleted := false + storageType := storageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Delete(time.Since(startedAt), deleted, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -30,15 +38,15 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { saddr := addr.EncodeToString() - // Check disk cache. - var has int + var dataSize int _ = c.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - has = len(b.Get([]byte(saddr))) + dataSize = len(b.Get([]byte(saddr))) return nil }) - if 0 < has { + if dataSize > 0 { + storageType = storageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -52,10 +60,12 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) + deleted = true c.objCounters.DecDB() return nil } + storageType = storageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, @@ -64,6 +74,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("fstree DELETE"), ) c.objCounters.DecFS() + deleted = true } return err diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index c6c8a9465..09c5451ad 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -199,7 +199,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data) + err = c.flushObject(ctx, &obj, data, storageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -228,7 +228,7 @@ func (c *cache) workerFlushSmall() { return } - err := c.flushObject(context.TODO(), obj, nil) + err := c.flushObject(context.TODO(), obj, nil, storageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -239,7 +239,13 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st storageType) error { + var err error + + defer func() { + c.metrics.Flush(err == nil, st) + }() + addr := objectCore.AddressOf(obj) var prm common.PutPrm @@ -313,7 +319,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data); err != nil { + if err := c.flushObject(ctx, &obj, data, storageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 030f9b413..f8eb01091 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -2,6 +2,7 @@ package writecache import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -27,9 +28,22 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() + return c.getInternal(ctx, saddr, addr) +} + +func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { + found := false + storageType := storageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Get(time.Since(startedAt), found, storageType) + }() + value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() + found = true + storageType = storageTypeDB return obj, obj.Unmarshal(value) } @@ -38,6 +52,8 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } + found = true + storageType = storageTypeFSTree return res.Object, nil } @@ -45,13 +61,15 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + saddr := addr.EncodeToString() + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head", trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), + attribute.String("address", saddr), )) defer span.End() - obj, err := c.Get(ctx, addr) + obj, err := c.getInternal(ctx, saddr, addr) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go new file mode 100644 index 000000000..afa6f69f1 --- /dev/null +++ b/pkg/local_object_storage/writecache/metrics.go @@ -0,0 +1,42 @@ +package writecache + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type storageType string + +const ( + storageTypeUndefined storageType = "null" + storageTypeDB storageType = "db" + storageTypeFSTree storageType = "fstree" +) + +type Metrics interface { + Get(d time.Duration, success bool, st storageType) + Delete(d time.Duration, success bool, st storageType) + Put(d time.Duration, success bool, st storageType) + Flush(success bool, st storageType) + Evict(st storageType) + + Estimate(db, fstree uint64) + SetMode(m mode.Mode) +} + +type metricsStub struct{} + +func (s *metricsStub) Get(time.Duration, bool, storageType) {} + +func (s *metricsStub) Delete(time.Duration, bool, storageType) {} + +func (s *metricsStub) Put(time.Duration, bool, storageType) {} + +func (s *metricsStub) Estimate(uint64, uint64) {} + +func (s *metricsStub) SetMode(mode.Mode) {} + +func (s *metricsStub) Flush(bool, storageType) {} + +func (s *metricsStub) Evict(storageType) {} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index ca6faff4c..7e9373a42 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -29,7 +29,11 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() - return c.setMode(ctx, m) + err := c.setMode(ctx, m) + if err == nil { + c.metrics.SetMode(m) + } + return err } // setMode applies new mode. Must be called with cache.modeMtx lock taken. diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 3434e9355..bea40aa36 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -60,6 +60,8 @@ type options struct { reportError func(string, error) // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. openFile func(string, int, fs.FileMode) (*os.File, error) + // metrics is metrics implementation + metrics Metrics } // WithLogger sets logger. @@ -164,3 +166,10 @@ func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { o.openFile = f } } + +// WithMetrics sets metrics implementation. +func WithMetrics(metrics Metrics) Option { + return func(o *options) { + o.metrics = metrics + } +} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 04d818b31..c0001f926 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -3,6 +3,7 @@ package writecache import ( "context" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -33,6 +34,13 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro )) defer span.End() + startedAt := time.Now() + added := false + storageType := storageTypeUndefined + defer func() { + c.metrics.Put(time.Since(startedAt), added, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -51,9 +59,20 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - return common.PutRes{}, c.putSmall(oi) + storageType = storageTypeDB + err := c.putSmall(oi) + if err == nil { + added = true + } + return common.PutRes{}, err } - return common.PutRes{}, c.putBig(ctx, oi.addr, prm) + + storageType = storageTypeFSTree + err := c.putBig(ctx, oi.addr, prm) + if err == nil { + added = true + } + return common.PutRes{}, err } // putSmall persists small objects to the write-cache database and diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 9c1c562b0..5f3092526 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -9,7 +9,10 @@ import ( ) func (c *cache) estimateCacheSize() uint64 { - return c.objCounters.DB()*c.smallObjectSize + c.objCounters.FS()*c.maxObjectSize + db := c.objCounters.DB() * c.smallObjectSize + fstree := c.objCounters.FS() * c.maxObjectSize + c.metrics.Estimate(db, fstree) + return db + fstree } func (c *cache) incSizeDB(sz uint64) uint64 { diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index c06d16c0b..50c110c5c 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -79,6 +79,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() + c.metrics.Evict(storageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -121,6 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) + c.metrics.Evict(storageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 0edf4b9be..664beff80 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -103,6 +103,7 @@ func New(opts ...Option) Cache { maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, openFile: os.OpenFile, + metrics: &metricsStub{}, }, } @@ -140,6 +141,7 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { + c.metrics.SetMode(c.mode) c.runFlushLoop() return nil } -- 2.45.2 From 2ce43935f95e1e6f665affeae9bf4cdcf81e8fff Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 19 May 2023 11:17:19 +0300 Subject: [PATCH 058/233] [#312] metrics: Add writecache metrcis Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 4 + pkg/local_object_storage/engine/shards.go | 7 + pkg/local_object_storage/engine/writecache.go | 53 ++++ pkg/local_object_storage/shard/shard.go | 7 + pkg/local_object_storage/writecache/delete.go | 6 +- pkg/local_object_storage/writecache/flush.go | 8 +- pkg/local_object_storage/writecache/get.go | 6 +- .../writecache/metrics.go | 39 +-- pkg/local_object_storage/writecache/put.go | 6 +- pkg/local_object_storage/writecache/state.go | 3 +- .../writecache/storage.go | 4 +- pkg/metrics/desc.go | 38 +++ pkg/metrics/node.go | 14 + pkg/metrics/writecache.go | 252 ++++++++++++++++++ 14 files changed, 415 insertions(+), 32 deletions(-) create mode 100644 pkg/metrics/writecache.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 13dcdfe02..7a11888c5 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -2,6 +2,8 @@ package engine import ( "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type MetricRegister interface { @@ -24,6 +26,8 @@ type MetricRegister interface { AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) + + WriteCache() metrics.WriteCacheMetrics } func elapsed(addFunc func(d time.Duration)) func() { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index e16d2c498..6c49c8312 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" @@ -98,6 +99,12 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { mw: e.metrics, }, )) + opts = append(opts, shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }, + ))) } e.mtx.RUnlock() diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 4effb2b16..ea2299b8b 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -2,9 +2,13 @@ package engine import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -52,3 +56,52 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } + +type writeCacheMetrics struct { + shardID string + metrics metrics.WriteCacheMetrics +} + +func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddGetDuration(m.shardID, success, d) + m.metrics.IncGetCounter(m.shardID, success, st.String()) +} + +func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddDeleteDuration(m.shardID, success, d) + m.metrics.IncDeleteCounter(m.shardID, success, st.String()) + if success { + m.metrics.DecActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddPutDuration(m.shardID, success, d) + m.metrics.IncPutCounter(m.shardID, success, st.String()) + if success { + m.metrics.IncActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { + m.metrics.SetEstimateSize(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetEstimateSize(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) SetMode(mode mode.Mode) { + m.metrics.SetMode(m.shardID, mode.String()) +} + +func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { + m.metrics.SetActualCount(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetActualCount(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { + m.metrics.IncFlushCounter(m.shardID, success, st.String()) +} + +func (m *writeCacheMetrics) Evict(st writecache.StorageType) { + m.metrics.DecActualCount(m.shardID, st.String()) + m.metrics.IncEvictCounter(m.shardID, st.String()) +} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 44ec54645..e0059105f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -187,6 +187,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option { } } +// WithExtraWriteCacheOptions returns option to add extra write cache options. +func WithExtraWriteCacheOptions(opts ...writecache.Option) Option { + return func(c *cfg) { + c.writeCacheOpts = append(c.writeCacheOpts, opts...) + } +} + // WithPiloramaOptions returns option to set internal write cache options. func WithPiloramaOptions(opts ...pilorama.Option) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index ed44a8ad8..f5a292ed4 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -24,7 +24,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { defer span.End() deleted := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Delete(time.Since(startedAt), deleted, storageType) @@ -46,7 +46,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { }) if dataSize > 0 { - storageType = storageTypeDB + storageType = StorageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -65,7 +65,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return nil } - storageType = storageTypeFSTree + storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 09c5451ad..28409f609 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -199,7 +199,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data, storageTypeFSTree) + err = c.flushObject(ctx, &obj, data, StorageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -228,7 +228,7 @@ func (c *cache) workerFlushSmall() { return } - err := c.flushObject(context.TODO(), obj, nil, storageTypeDB) + err := c.flushObject(context.TODO(), obj, nil, StorageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -239,7 +239,7 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st storageType) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st StorageType) error { var err error defer func() { @@ -319,7 +319,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data, storageTypeDB); err != nil { + if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index f8eb01091..f8f6de9b0 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -33,7 +33,7 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { found := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Get(time.Since(startedAt), found, storageType) @@ -43,7 +43,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) if err == nil { obj := objectSDK.New() found = true - storageType = storageTypeDB + storageType = StorageTypeDB return obj, obj.Unmarshal(value) } @@ -53,7 +53,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) } found = true - storageType = storageTypeFSTree + storageType = StorageTypeFSTree return res.Object, nil } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index afa6f69f1..e20c7e65e 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -6,37 +6,44 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) -type storageType string +type StorageType string + +func (t StorageType) String() string { + return string(t) +} const ( - storageTypeUndefined storageType = "null" - storageTypeDB storageType = "db" - storageTypeFSTree storageType = "fstree" + StorageTypeUndefined StorageType = "null" + StorageTypeDB StorageType = "db" + StorageTypeFSTree StorageType = "fstree" ) type Metrics interface { - Get(d time.Duration, success bool, st storageType) - Delete(d time.Duration, success bool, st storageType) - Put(d time.Duration, success bool, st storageType) - Flush(success bool, st storageType) - Evict(st storageType) + Get(d time.Duration, success bool, st StorageType) + Delete(d time.Duration, success bool, st StorageType) + Put(d time.Duration, success bool, st StorageType) + Flush(success bool, st StorageType) + Evict(st StorageType) - Estimate(db, fstree uint64) + SetEstimateSize(db, fstree uint64) SetMode(m mode.Mode) + SetActualCounters(db, fstree uint64) } type metricsStub struct{} -func (s *metricsStub) Get(time.Duration, bool, storageType) {} +func (s *metricsStub) Get(time.Duration, bool, StorageType) {} -func (s *metricsStub) Delete(time.Duration, bool, storageType) {} +func (s *metricsStub) Delete(time.Duration, bool, StorageType) {} -func (s *metricsStub) Put(time.Duration, bool, storageType) {} +func (s *metricsStub) Put(time.Duration, bool, StorageType) {} -func (s *metricsStub) Estimate(uint64, uint64) {} +func (s *metricsStub) SetEstimateSize(uint64, uint64) {} func (s *metricsStub) SetMode(mode.Mode) {} -func (s *metricsStub) Flush(bool, storageType) {} +func (s *metricsStub) SetActualCounters(uint64, uint64) {} -func (s *metricsStub) Evict(storageType) {} +func (s *metricsStub) Flush(bool, StorageType) {} + +func (s *metricsStub) Evict(StorageType) {} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index c0001f926..1e99e4a28 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -36,7 +36,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro startedAt := time.Now() added := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined defer func() { c.metrics.Put(time.Since(startedAt), added, storageType) }() @@ -59,7 +59,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - storageType = storageTypeDB + storageType = StorageTypeDB err := c.putSmall(oi) if err == nil { added = true @@ -67,7 +67,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro return common.PutRes{}, err } - storageType = storageTypeFSTree + storageType = StorageTypeFSTree err := c.putBig(ctx, oi.addr, prm) if err == nil { added = true diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 5f3092526..14103e626 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -11,7 +11,7 @@ import ( func (c *cache) estimateCacheSize() uint64 { db := c.objCounters.DB() * c.smallObjectSize fstree := c.objCounters.FS() * c.maxObjectSize - c.metrics.Estimate(db, fstree) + c.metrics.SetEstimateSize(db, fstree) return db + fstree } @@ -71,6 +71,7 @@ func (c *cache) initCounters() error { c.objCounters.cDB.Store(inDB) c.objCounters.cFS.Store(inFS) + c.metrics.SetActualCounters(inDB, inFS) return nil } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 50c110c5c..3bd3813d1 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -79,7 +79,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() - c.metrics.Evict(storageTypeDB) + c.metrics.Evict(StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -122,7 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) - c.metrics.Evict(storageTypeFSTree) + c.metrics.Evict(StorageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go index 74d2d4e6e..612435b2f 100644 --- a/pkg/metrics/desc.go +++ b/pkg/metrics/desc.go @@ -48,6 +48,18 @@ func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometh } } +func newGaugeFunc(opts prometheus.GaugeOpts, f func() float64) metric[prometheus.GaugeFunc] { + return metric[prometheus.GaugeFunc]{ + value: prometheus.NewGaugeFunc(opts, f), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { return metric[prometheus.Counter]{ value: prometheus.NewCounter(opts), @@ -60,6 +72,32 @@ func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { } } +func newCounterVec(opts prometheus.CounterOpts, labels []string) metric[*prometheus.CounterVec] { + return metric[*prometheus.CounterVec]{ + value: prometheus.NewCounterVec(opts, labels), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_COUNTER.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labels, + }, + } +} + +func newHistogramVec(opts prometheus.HistogramOpts, labelNames []string) metric[*prometheus.HistogramVec] { + return metric[*prometheus.HistogramVec]{ + value: prometheus.NewHistogramVec(opts, labelNames), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_HISTOGRAM.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labelNames, + }, + } +} + // DescribeAll returns descriptions for all registered metrics. func DescribeAll() ([]Description, error) { registeredDescriptionsMtx.Lock() diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index bf12e610f..b8041eec8 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -10,6 +10,8 @@ type NodeMetrics struct { stateMetrics replicatorMetrics epoch metric[prometheus.Gauge] + + writeCacheMetrics *writeCacheMetrics } func NewNodeMetrics() *NodeMetrics { @@ -33,12 +35,16 @@ func NewNodeMetrics() *NodeMetrics { }) mustRegister(epoch) + writeCacheMetrics := newWriteCacheMetrics() + writeCacheMetrics.register() + return &NodeMetrics{ objectServiceMetrics: objectService, engineMetrics: engine, stateMetrics: state, replicatorMetrics: replicator, epoch: epoch, + writeCacheMetrics: writeCacheMetrics, } } @@ -46,3 +52,11 @@ func NewNodeMetrics() *NodeMetrics { func (m *NodeMetrics) SetEpoch(epoch uint64) { m.epoch.value.Set(float64(epoch)) } + +// WriteCache returns WriteCache metrics. +func (m *NodeMetrics) WriteCache() WriteCacheMetrics { + if m == nil { + return nil + } + return m.writeCacheMetrics +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go new file mode 100644 index 000000000..890903546 --- /dev/null +++ b/pkg/metrics/writecache.go @@ -0,0 +1,252 @@ +package metrics + +import ( + "fmt" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + wcSubsystem = "writecache" + wcShardID = "shard_id" + wcSuccess = "success" + wcStorage = "storage" + wcMode = "mode" +) + +type shardIDMode struct { + shardID, mode string +} + +type WriteCacheMetrics interface { + AddGetDuration(shardID string, success bool, d time.Duration) + IncGetCounter(shardID string, success bool, storageType string) + + AddDeleteDuration(shardID string, success bool, d time.Duration) + IncDeleteCounter(shardID string, success bool, storageType string) + + AddPutDuration(shardID string, success bool, d time.Duration) + IncPutCounter(shardID string, success bool, storageType string) + + IncActualCount(shardID string, storageType string) + DecActualCount(shardID string, storageType string) + SetActualCount(shardID string, count uint64, storageType string) + + SetEstimateSize(shardID string, size uint64, storageType string) + SetMode(shardID string, mode string) + + IncFlushCounter(shardID string, success bool, storageType string) + IncEvictCounter(shardID string, storageType string) +} + +type writeCacheMetrics struct { + getDuration metric[*prometheus.HistogramVec] + getCounter metric[*prometheus.CounterVec] + + putDuration metric[*prometheus.HistogramVec] + putCounter metric[*prometheus.CounterVec] + + deleteDuration metric[*prometheus.HistogramVec] + deleteCounter metric[*prometheus.CounterVec] + + flushCounter metric[*prometheus.CounterVec] + evictCounter metric[*prometheus.CounterVec] + + actualCount metric[*prometheus.GaugeVec] + + estimatedSize metric[*prometheus.GaugeVec] + + modeMetrics map[shardIDMode]metric[prometheus.GaugeFunc] + modeValues map[string]string + modeMtx sync.RWMutex +} + +func newWriteCacheMetrics() *writeCacheMetrics { + return &writeCacheMetrics{ + getDuration: newWCMethodDurationCounter("get"), + getCounter: newWCMethodCounterVec("get"), + putDuration: newWCMethodDurationCounter("put"), + putCounter: newWCMethodCounterVec("put"), + deleteDuration: newWCMethodDurationCounter("delete"), + deleteCounter: newWCMethodCounterVec("delete"), + flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), + evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), + actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), + modeMtx: sync.RWMutex{}, + modeMetrics: make(map[shardIDMode]metric[prometheus.GaugeFunc]), + modeValues: make(map[string]string), + } +} + +func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.getDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncGetCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.getCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.deleteDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncDeleteCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.deleteCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.putDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncPutCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.putCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Dec() +} + +func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Set(float64(count)) +} + +func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { + m.estimatedSize.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Set(float64(size)) +} + +func (m *writeCacheMetrics) SetMode(shardID string, mode string) { + m.modeMtx.Lock() + defer m.modeMtx.Unlock() + + m.modeValues[shardID] = mode + key := shardIDMode{ + shardID: shardID, + mode: mode, + } + if _, found := m.modeMetrics[key]; found { + return + } + + metric := newGaugeFunc( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "writecache_mode", + Help: "Writecache mode value", + ConstLabels: prometheus.Labels{ + wcShardID: shardID, + wcMode: mode, + }, + }, func() float64 { + m.modeMtx.RLock() + defer m.modeMtx.RUnlock() + + value := m.modeValues[shardID] + if value == mode { + return 1 + } + return 0 + }) + mustRegister(metric) + m.modeMetrics[key] = metric +} + +func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { + m.flushCounter.value.With(prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { + m.evictCounter.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) register() { + mustRegister(m.getDuration) + mustRegister(m.getCounter) + mustRegister(m.putDuration) + mustRegister(m.putCounter) + mustRegister(m.deleteDuration) + mustRegister(m.deleteCounter) + mustRegister(m.actualCount) + mustRegister(m.estimatedSize) + mustRegister(m.flushCounter) + mustRegister(m.evictCounter) +} + +func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration) { + m.With( + prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func incWriteCacheCounter(m *prometheus.CounterVec, shardID string, success bool, storageType string) { + m.With(prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + }).Inc() +} + +func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { + return newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_req_duration_seconds", method), + Help: fmt.Sprintf("Accumulated %s request process duration", method), + }, []string{wcShardID, wcSuccess}) +} + +func newWCMethodCounterVec(method string) metric[*prometheus.CounterVec] { + return newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_req_count", method), + Help: fmt.Sprintf("The number of %s requests processed", method), + }, []string{wcShardID, wcSuccess, wcStorage}) +} + +func newWCOperationCounterVec(operation string, labels []string) metric[*prometheus.CounterVec] { + return newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_operation_count", operation), + Help: fmt.Sprintf("The number of %s operations processed", operation), + }, labels) +} + +func newWCGaugeVec(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: name, + Help: help, + }, labels) +} -- 2.45.2 From a1823423cf238d9c8634111fd9203e8c1f8aa940 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 22 May 2023 10:10:25 +0300 Subject: [PATCH 059/233] [#312] node: Add WC metrics info to CHANGELOG Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f79940f76..b5924b781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changelog for FrostFS Node - Reload pprof and metrics on SIGHUP for ir (#125) - Support copies number parameter in `frostfs-cli object put` (#351) - Set extra wallets on SIGHUP for ir (#125) +- Writecache metrics (#312) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) -- 2.45.2 From 261335100845817f99133eac2bb5d658c3cd3d9e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 15:02:38 +0300 Subject: [PATCH 060/233] [#387] gc: Cancel GC is change mode requested Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/shard/control.go | 16 +- pkg/local_object_storage/shard/delete.go | 120 ++++++++------- pkg/local_object_storage/shard/gc.go | 16 +- .../shard/gc_internal_test.go | 144 ++++++++++++++++++ pkg/local_object_storage/shard/mode.go | 4 +- pkg/local_object_storage/shard/shard.go | 8 +- 7 files changed, 244 insertions(+), 65 deletions(-) create mode 100644 pkg/local_object_storage/shard/gc_internal_test.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index b84746a72..a862e745a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -489,4 +489,5 @@ const ( EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" + ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" ) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e74f235f8..f885451af 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -304,8 +304,8 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { opts[i](&c) } - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { @@ -335,3 +335,15 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { s.log.Info(logs.ShardTryingToRestoreReadwriteMode) return s.setMode(mode.ReadWrite) } + +func (s *Shard) lockExclusive() func() { + s.setModeRequested.Store(true) + val := s.gcCancel.Load() + if val != nil { + cancelGC := val.(context.CancelFunc) + cancelGC() + } + s.m.Lock() + s.setModeRequested.Store(false) + return s.m.Unlock +} diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index f086aa30f..4eb7ad6af 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -53,70 +53,74 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrDegradedMode } - ln := len(prm.addr) - - smalls := make(map[oid.Address][]byte, ln) - - for i := range prm.addr { - if s.hasWriteCache() { - err := s.writeCache.Delete(ctx, prm.addr[i]) - if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { - s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) - } + for _, addr := range prm.addr { + select { + case <-ctx.Done(): + return DeleteRes{}, ctx.Err() + default: } - var sPrm meta.StorageIDPrm - sPrm.SetAddress(prm.addr[i]) + s.deleteObjectFromWriteCacheSafe(ctx, addr) - res, err := s.metaBase.StorageID(ctx, sPrm) - if err != nil { - s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, - zap.Stringer("object", prm.addr[i]), - zap.String("error", err.Error())) + s.deleteFromBlobstorSafe(ctx, addr) - continue - } - - if res.StorageID() != nil { - smalls[prm.addr[i]] = res.StorageID() - } - } - - var delPrm meta.DeletePrm - delPrm.SetAddresses(prm.addr...) - - res, err := s.metaBase.Delete(ctx, delPrm) - if err != nil { - return DeleteRes{}, err // stop on metabase error ? - } - - var totalRemovedPayload uint64 - - s.decObjectCounterBy(physical, res.RawObjectsRemoved()) - s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) - for i := range prm.addr { - removedPayload := res.RemovedPhysicalObjectSizes()[i] - totalRemovedPayload += removedPayload - logicalRemovedPayload := res.RemovedLogicalObjectSizes()[i] - if logicalRemovedPayload > 0 { - s.addToContainerSize(prm.addr[i].Container().EncodeToString(), -int64(logicalRemovedPayload)) - } - } - s.addToPayloadSize(-int64(totalRemovedPayload)) - - for i := range prm.addr { - var delPrm common.DeletePrm - delPrm.Address = prm.addr[i] - id := smalls[prm.addr[i]] - delPrm.StorageID = id - - _, err = s.blobStor.Delete(ctx, delPrm) - if err != nil { - s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, - zap.Stringer("object_address", prm.addr[i]), - zap.String("error", err.Error())) + if err := s.deleteFromMetabase(ctx, addr); err != nil { + return DeleteRes{}, err // stop on metabase error ? } } return DeleteRes{}, nil } + +func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { + if s.hasWriteCache() { + err := s.writeCache.Delete(ctx, addr) + if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { + s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err)) + } + } +} + +func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { + var sPrm meta.StorageIDPrm + sPrm.SetAddress(addr) + + res, err := s.metaBase.StorageID(ctx, sPrm) + if err != nil { + s.log.Debug("can't get storage ID from metabase", + zap.Stringer("object", addr), + zap.String("error", err.Error())) + } + storageID := res.StorageID() + + var delPrm common.DeletePrm + delPrm.Address = addr + delPrm.StorageID = storageID + + _, err = s.blobStor.Delete(ctx, delPrm) + if err != nil { + s.log.Debug("can't remove object from blobStor", + zap.Stringer("object_address", addr), + zap.String("error", err.Error())) + } +} + +func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error { + var delPrm meta.DeletePrm + delPrm.SetAddresses(addr) + + res, err := s.metaBase.Delete(ctx, delPrm) + if err != nil { + return err + } + s.decObjectCounterBy(physical, res.RawObjectsRemoved()) + s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) + removedPayload := res.RemovedPhysicalObjectSizes()[0] + logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] + if logicalRemovedPayload > 0 { + s.addToContainerSize(addr.Container().EncodeToString(), -int64(logicalRemovedPayload)) + } + s.addToPayloadSize(-int64(removedPayload)) + + return nil +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index f4f7a21e2..97899acc2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -197,6 +197,14 @@ func (gc *gc) stop() { // with GC-marked graves. // Does nothing if shard is in "read-only" mode. func (s *Shard) removeGarbage() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.gcCancel.Store(cancel) + if s.setModeRequested.Load() { + return + } + s.m.RLock() defer s.m.RUnlock() @@ -211,6 +219,12 @@ func (s *Shard) removeGarbage() { var iterPrm meta.GarbageIterationPrm iterPrm.SetHandler(func(g meta.GarbageObject) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + buf = append(buf, g.Address()) if len(buf) == s.rmBatchSize { @@ -237,7 +251,7 @@ func (s *Shard) removeGarbage() { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(context.TODO(), deletePrm) + _, err = s.delete(ctx, deletePrm) if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go new file mode 100644 index 000000000..d94029976 --- /dev/null +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -0,0 +1,144 @@ +package shard + +import ( + "context" + "path/filepath" + "testing" + "time" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { + t.Parallel() + + rootPath := t.TempDir() + + var sh *Shard + + l := &logger.Logger{Logger: zaptest.NewLogger(t)} + blobOpts := []blobstor.Option{ + blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *object.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + } + + opts := []Option{ + WithID(NewIDFromBytes([]byte{})), + WithLogger(l), + WithBlobStorOptions(blobOpts...), + WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epochState{}), + ), + WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), + WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + WithGCRemoverSleepInterval(1 * time.Second), + } + + sh = New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init(context.Background())) + + t.Cleanup(func() { + require.NoError(t, sh.Close()) + }) + + cnr := cidtest.ID() + obj := testutil.GenerateObjectWithCID(cnr) + objID, _ := obj.ID() + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(objID) + + var putPrm PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm GetPrm + getPrm.SetAddress(objectCore.AddressOf(obj)) + _, err = sh.Get(context.Background(), getPrm) + require.NoError(t, err, "failed to get") + + //inhume + var inhumePrm InhumePrm + inhumePrm.MarkAsGarbage(addr) + _, err = sh.Inhume(context.Background(), inhumePrm) + require.NoError(t, err, "failed to inhume") + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned error") + require.True(t, IsErrNotFound(err), "invalid error type") + + //storageID + var metaStIDPrm meta.StorageIDPrm + metaStIDPrm.SetAddress(addr) + storageID, err := sh.metaBase.StorageID(context.Background(), metaStIDPrm) + require.NoError(t, err, "failed to get storage ID") + + //check existance in blobstore + var bsExisted common.ExistsPrm + bsExisted.Address = addr + bsExisted.StorageID = storageID.StorageID() + exRes, err := sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.True(t, exRes.Exists, "invalid blobstore existance result") + + //drop from blobstor + var bsDeletePrm common.DeletePrm + bsDeletePrm.Address = addr + bsDeletePrm.StorageID = storageID.StorageID() + _, err = sh.blobStor.Delete(context.Background(), bsDeletePrm) + require.NoError(t, err, "failed to delete from blobstore") + + //check existance in blobstore + exRes, err = sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.False(t, exRes.Exists, "invalid blobstore existance result") + + //get should return object not found + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned no error") + require.True(t, IsErrNotFound(err), "invalid error type") +} diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 50c52accc..efd41863b 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -19,8 +19,8 @@ var ErrDegradedMode = logicerr.New("shard is in degraded mode") // Returns any error encountered that did not allow // setting shard mode. func (s *Shard) SetMode(m mode.Mode) error { - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() return s.setMode(m) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index e0059105f..65cc1ef55 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -3,6 +3,7 @@ package shard import ( "context" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -32,6 +33,9 @@ type Shard struct { metaBase *meta.DB tsSource TombstoneSource + + gcCancel atomic.Value + setModeRequested atomic.Bool } // Option represents Shard's constructor option. @@ -217,12 +221,12 @@ func WithWriteCache(use bool) Option { } // hasWriteCache returns bool if write cache exists on shards. -func (s Shard) hasWriteCache() bool { +func (s *Shard) hasWriteCache() bool { return s.cfg.useWriteCache } // needRefillMetabase returns true if metabase is needed to be refilled. -func (s Shard) needRefillMetabase() bool { +func (s *Shard) needRefillMetabase() bool { return s.cfg.refillMetabase } -- 2.45.2 From 9119199f6ed0ad7c8059132b5b9c506df1f8d0b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 26 May 2023 11:26:54 +0300 Subject: [PATCH 061/233] [#397] cli: Fix evacuation method names Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 8 ++-- pkg/services/control/rpc.go | 42 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 665bcf85f..45152fa0a 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -67,7 +67,7 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { var resp *control.StartShardEvacuationResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.StartEvacuateShard(client, req) + resp, err = control.StartShardEvacuation(client, req) return err }) commonCmd.ExitOnErr(cmd, "Start evacuate shards failed, rpc error: %w", err) @@ -95,7 +95,7 @@ func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { var resp *control.GetShardEvacuationStatusResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.GetEvacuateShardStatus(client, req) + resp, err = control.GetShardEvacuationStatus(client, req) return err }) commonCmd.ExitOnErr(cmd, "Get evacuate shards status failed, rpc error: %w", err) @@ -118,7 +118,7 @@ func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { var resp *control.StopShardEvacuationResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.StopEvacuateShard(client, req) + resp, err = control.StopShardEvacuation(client, req) return err }) commonCmd.ExitOnErr(cmd, "Stop evacuate shards failed, rpc error: %w", err) @@ -166,7 +166,7 @@ func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clien var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.GetEvacuateShardStatus(client, req) + resp, err = control.GetShardEvacuationStatus(client, req) return err }) diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 3822c9f70..a2e7c411a 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,18 +8,18 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcStartEvacuateShard = "StartEvacuateShard" - rpcGetEvacuateShardStatus = "GetEvacuateShardStatus" - rpcStopEvacuateShardStatus = "StopEvacuateShard" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -144,12 +144,12 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return wResp.message, nil } -// StartEvacuateShard executes ControlService.StartEvacuateShard RPC. -func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { +// StartShardEvacuation executes ControlService.StartShardEvacuation RPC. +func StartShardEvacuation(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { wResp := newResponseWrapper[StartShardEvacuationResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartEvacuateShard), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartShardEvacuation), wReq, wResp, opts...) if err != nil { return nil, err } @@ -157,12 +157,12 @@ func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, op return wResp.message, nil } -// GetEvacuateShardStatus executes ControlService.GetEvacuateShardStatus RPC. -func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { +// GetShardEvacuationStatus executes ControlService.GetShardEvacuationStatus RPC. +func GetShardEvacuationStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { wResp := newResponseWrapper[GetShardEvacuationStatusResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetEvacuateShardStatus), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetShardEvacuationStatus), wReq, wResp, opts...) if err != nil { return nil, err } @@ -170,12 +170,12 @@ func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusReq return wResp.message, nil } -// StopEvacuateShard executes ControlService.StopEvacuateShard RPC. -func StopEvacuateShard(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { +// StopShardEvacuation executes ControlService.StopShardEvacuation RPC. +func StopShardEvacuation(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { wResp := newResponseWrapper[StopShardEvacuationResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopEvacuateShardStatus), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopShardEvacuation), wReq, wResp, opts...) if err != nil { return nil, err } -- 2.45.2 From 20a489bdb58b595084be6240de8967205e09eca8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 16:12:46 +0300 Subject: [PATCH 062/233] [#393] gc: Use defer to mark handler done Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 97899acc2..78c41cb64 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -146,8 +146,8 @@ func (gc *gc) listenEvents(ctx context.Context) { h := v.handlers[i] err := gc.workerPool.Submit(func() { + defer v.prevGroup.Done() h(runCtx, event) - v.prevGroup.Done() }) if err != nil { gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, -- 2.45.2 From 598361706969c697c18826fd6d85005e5b905cea Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 16:20:39 +0300 Subject: [PATCH 063/233] [#393] shard: Create tombstone source when reload Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index fe2b5ad35..30d457967 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -847,18 +847,9 @@ func initLocalStorage(c *cfg) { // service will be created later c.cfgObject.getSvc = new(getsvc.Service) - var tssPrm tsourse.TombstoneSourcePrm - tssPrm.SetGetService(c.cfgObject.getSvc) - tombstoneSrc := tsourse.NewSource(tssPrm) - - tombstoneSource := tombstone.NewChecker( - tombstone.WithLogger(c.log), - tombstone.WithTombstoneSource(tombstoneSrc), - ) - var shardsAttached int for _, optsWithMeta := range c.shardOpts() { - id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...) + id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { @@ -1059,7 +1050,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { var rcfg engine.ReConfiguration for _, optsWithID := range c.shardOpts() { - rcfg.AddShard(optsWithID.configID, optsWithID.shOpts) + rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) } err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) @@ -1080,6 +1071,18 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } +func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { + var tssPrm tsourse.TombstoneSourcePrm + tssPrm.SetGetService(c.cfgObject.getSvc) + tombstoneSrc := tsourse.NewSource(tssPrm) + + tombstoneSource := tombstone.NewChecker( + tombstone.WithLogger(c.log), + tombstone.WithTombstoneSource(tombstoneSrc), + ) + return tombstoneSource +} + func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) -- 2.45.2 From f2e5dead7e9f893fe4a7de807480e999aed74a57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 25 May 2023 18:40:27 +0300 Subject: [PATCH 064/233] [#398] pilorama: Disallow applying same operations 1. In redo() we save the old state. 2. If we do redo() for the same operation twice, the old state will be overritten with the new one. 3. This in turn affects undo() and subsequent isAncestor() check. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 15 +++- .../pilorama/batch_test.go | 70 +++++++++++++++++ .../pilorama/forest_test.go | 78 +++++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 pkg/local_object_storage/pilorama/batch_test.go diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 3065c8370..5722c68aa 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -50,10 +50,23 @@ func (b *batch) run() { return b.operations[i].Time < b.operations[j].Time }) + b.operations = removeDuplicatesInPlace(b.operations) var lm Move return b.forest.applyOperation(bLog, bTree, b.operations, &lm) }) - for i := range b.operations { + for i := range b.results { b.results[i] <- err } } + +func removeDuplicatesInPlace(a []*Move) []*Move { + equalCount := 0 + for i := 1; i < len(a); i++ { + if a[i].Time == a[i-1].Time { + equalCount++ + } else { + a[i-equalCount] = a[i] + } + } + return a[:len(a)-equalCount] +} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go new file mode 100644 index 000000000..931fce18c --- /dev/null +++ b/pkg/local_object_storage/pilorama/batch_test.go @@ -0,0 +1,70 @@ +package pilorama + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_removeDuplicatesInPlace(t *testing.T) { + testCases := []struct { + before []int + after []int + }{ + { + before: []int{}, + after: []int{}, + }, + { + before: []int{1}, + after: []int{1}, + }, + { + before: []int{1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 1}, + after: []int{1}, + }, + { + before: []int{1, 1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 1, 1, 2, 3, 3, 3}, + after: []int{1, 2, 3}, + }, + } + + for _, tc := range testCases { + ops := make([]*Move, len(tc.before)) + for i := range ops { + ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} + } + + expected := make([]*Move, len(tc.after)) + for i := range expected { + expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} + } + + actual := removeDuplicatesInPlace(ops) + require.Equal(t, expected, actual, "%d", tc.before) + } +} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 0cff28c3f..ebb4667f5 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -13,6 +13,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var providers = []struct { @@ -444,6 +445,83 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio }) } +func TestForest_ApplySameOperation(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + parallel := providers[i].name != "inmemory" + testForestApplySameOperation(t, providers[i].construct, parallel) + }) + } +} + +func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, parallel bool) { + cid := cidtest.ID() + treeID := "version" + + batchSize := 3 + ctx := context.Background() + errG, _ := errgroup.WithContext(ctx) + if !parallel { + batchSize = 1 + errG.SetLimit(1) + } + + meta := []Meta{ + {Time: 1, Items: []KeyValue{{AttributeFilename, []byte("1")}, {"attr", []byte{1}}}}, + {Time: 2, Items: []KeyValue{{AttributeFilename, []byte("2")}, {"attr", []byte{1}}}}, + {Time: 3, Items: []KeyValue{{AttributeFilename, []byte("3")}, {"attr", []byte{1}}}}, + } + logs := []Move{ + { + Child: 1, + Parent: RootID, + Meta: meta[0], + }, + { + Child: 2, + Parent: 1, + Meta: meta[1], + }, + { + Child: 1, + Parent: 2, + Meta: meta[2], + }, + } + + check := func(t *testing.T, s Forest) { + testMeta(t, s, cid, treeID, 1, RootID, meta[0]) + testMeta(t, s, cid, treeID, 2, 1, meta[1]) + + nodes, err := s.TreeGetChildren(ctx, cid, treeID, RootID) + require.NoError(t, err) + require.Equal(t, []Node{1}, nodes) + + nodes, err = s.TreeGetChildren(ctx, cid, treeID, 1) + require.NoError(t, err) + require.Equal(t, []Node{2}, nodes) + } + + t.Run("expected", func(t *testing.T) { + s := constructor(t) + for i := range logs { + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[i], false)) + } + check(t, s) + }) + + s := constructor(t, WithMaxBatchSize(batchSize)) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) + for i := 0; i < batchSize; i++ { + errG.Go(func() error { + return s.TreeApply(ctx, cid, treeID, &logs[2], false) + }) + } + require.NoError(t, errG.Wait()) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[1], false)) + check(t, s) +} + func TestForest_GetOpLog(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { -- 2.45.2 From bc34fee6a78dfe916b1407d67ccad5416a1aa141 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 24 May 2023 10:01:50 +0300 Subject: [PATCH 065/233] [#370] Add tree service metrics Signed-off-by: Alejandro Lopez --- CHANGELOG.md | 1 + cmd/frostfs-node/tree.go | 3 +- pkg/metrics/node.go | 16 +++++++-- pkg/metrics/treeservice.go | 64 +++++++++++++++++++++++++++++++++ pkg/services/tree/metrics.go | 15 ++++++++ pkg/services/tree/options.go | 8 +++++ pkg/services/tree/replicator.go | 6 ++++ pkg/services/tree/service.go | 1 + pkg/services/tree/sync.go | 6 ++++ 9 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 pkg/metrics/treeservice.go create mode 100644 pkg/services/tree/metrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b5924b781..83ec8e41a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog for FrostFS Node - Support copies number parameter in `frostfs-cli object put` (#351) - Set extra wallets on SIGHUP for ir (#125) - Writecache metrics (#312) +- Add tree service metrics (#370) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index b4f43acac..fffaa01d1 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -55,7 +55,8 @@ func initTreeService(c *cfg) { tree.WithContainerCacheSize(treeConfig.CacheSize()), tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), - tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount())) + tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithMetrics(c.metricsCollector.TreeService())) for _, srv := range c.cfgGRPC.servers { tree.RegisterTreeServiceServer(srv, c.treeService) diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index b8041eec8..cca82b5fe 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,6 +1,9 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "github.com/prometheus/client_golang/prometheus" +) const namespace = "frostfs_node" @@ -9,9 +12,10 @@ type NodeMetrics struct { engineMetrics stateMetrics replicatorMetrics - epoch metric[prometheus.Gauge] writeCacheMetrics *writeCacheMetrics + treeService *treeServiceMetrics + epoch metric[prometheus.Gauge] } func NewNodeMetrics() *NodeMetrics { @@ -27,6 +31,9 @@ func NewNodeMetrics() *NodeMetrics { replicator := newReplicatorMetrics() replicator.register() + treeService := newTreeServiceMetrics() + treeService.register() + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, @@ -43,6 +50,7 @@ func NewNodeMetrics() *NodeMetrics { engineMetrics: engine, stateMetrics: state, replicatorMetrics: replicator, + treeService: treeService, epoch: epoch, writeCacheMetrics: writeCacheMetrics, } @@ -60,3 +68,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { } return m.writeCacheMetrics } + +func (m *NodeMetrics) TreeService() tree.MetricsRegister { + return m.treeService +} diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go new file mode 100644 index 000000000..135f6e6d2 --- /dev/null +++ b/pkg/metrics/treeservice.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const treeServiceLabelSuccess = "success" + +type treeServiceMetrics struct { + replicateTaskDuration metric[*prometheus.HistogramVec] + replicateWaitDuration metric[*prometheus.HistogramVec] + syncOpDuration metric[*prometheus.HistogramVec] +} + +func newTreeServiceMetrics() *treeServiceMetrics { + const treeServiceSubsystem = "treeservice" + return &treeServiceMetrics{ + replicateTaskDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_task_duration_seconds", + Help: "Duration of individual replication tasks executed as part of replication loops", + }, []string{treeServiceLabelSuccess}), + replicateWaitDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_wait_duration_seconds", + Help: "Duration of overall waiting time for replication loops", + }, []string{treeServiceLabelSuccess}), + syncOpDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "sync_duration_seconds", + Help: "Duration of synchronization operations", + }, []string{treeServiceLabelSuccess}), + } +} + +func (m *treeServiceMetrics) register() { + mustRegister(m.replicateTaskDuration) + mustRegister(m.replicateWaitDuration) + mustRegister(m.syncOpDuration) +} + +func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { + m.replicateTaskDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { + m.replicateWaitDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { + m.syncOpDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go new file mode 100644 index 000000000..0f0e4ee57 --- /dev/null +++ b/pkg/services/tree/metrics.go @@ -0,0 +1,15 @@ +package tree + +import "time" + +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + +type defaultMetricsRegister struct{} + +func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddReplicateWaitDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddSyncDuration(time.Duration, bool) {} diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index d60bc14c5..bcaf21f92 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,6 +33,8 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int + + metrics MetricsRegister } // Option represents configuration option for a tree service. @@ -116,3 +118,9 @@ func WithReplicationTimeout(t time.Duration) Option { } } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 60d0eff50..7199dc40e 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -75,6 +75,7 @@ func (s *Service) replicationWorker(ctx context.Context) { attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), ), ) + start := time.Now() var lastErr error var lastAddr string @@ -113,6 +114,9 @@ func (s *Service) replicationWorker(ctx context.Context) { zap.String("address", lastAddr), zap.String("key", hex.EncodeToString(task.n.PublicKey()))) } + s.metrics.AddReplicateTaskDuration(time.Since(start), false) + } else { + s.metrics.AddReplicateTaskDuration(time.Since(start), true) } span.End() } @@ -137,6 +141,7 @@ func (s *Service) replicateLoop(ctx context.Context) { case <-ctx.Done(): return case op := <-s.replicateCh: + start := time.Now() err := s.replicate(op) if err != nil { s.log.Error(logs.TreeErrorDuringReplication, @@ -144,6 +149,7 @@ func (s *Service) replicateLoop(ctx context.Context) { zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) } + s.metrics.AddReplicateWaitDuration(time.Since(start), err == nil) } } } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 546b7a207..96e547f36 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -46,6 +46,7 @@ func New(opts ...Option) *Service { s.replicatorChannelCapacity = defaultReplicatorCapacity s.replicatorWorkerCount = defaultReplicatorWorkerCount s.replicatorTimeout = defaultReplicatorSendTimeout + s.metrics = defaultMetricsRegister{} for i := range opts { opts[i](&s.cfg) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ed2455194..ec51c6bc6 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -9,6 +9,7 @@ import ( "math" "math/rand" "sync" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -376,9 +377,12 @@ func (s *Service) syncLoop(ctx context.Context) { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") s.log.Debug(logs.TreeSyncingTrees) + start := time.Now() + cnrs, err := s.cfg.cnrSource.List() if err != nil { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) + s.metrics.AddSyncDuration(time.Since(start), false) span.End() continue } @@ -390,6 +394,8 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) s.log.Debug(logs.TreeTreesHaveBeenSynchronized) + + s.metrics.AddSyncDuration(time.Since(start), true) span.End() } } -- 2.45.2 From 271a56c2ab6ef3796041b32ece3d6507e8dad022 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 17:46:24 +0300 Subject: [PATCH 066/233] [#395] metrics: Drop redundant metrics HistogramVec already has labeled counter. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 9 +-- pkg/metrics/writecache.go | 70 ++++--------------- 2 files changed, 16 insertions(+), 63 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index ea2299b8b..692fa4be5 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -63,21 +63,18 @@ type writeCacheMetrics struct { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddGetDuration(m.shardID, success, d) - m.metrics.IncGetCounter(m.shardID, success, st.String()) + m.metrics.AddGetDuration(m.shardID, success, d, st.String()) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddDeleteDuration(m.shardID, success, d) - m.metrics.IncDeleteCounter(m.shardID, success, st.String()) + m.metrics.AddDeleteDuration(m.shardID, success, d, st.String()) if success { m.metrics.DecActualCount(m.shardID, st.String()) } } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddPutDuration(m.shardID, success, d) - m.metrics.IncPutCounter(m.shardID, success, st.String()) + m.metrics.AddPutDuration(m.shardID, success, d, st.String()) if success { m.metrics.IncActualCount(m.shardID, st.String()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 890903546..74c330842 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -21,14 +21,9 @@ type shardIDMode struct { } type WriteCacheMetrics interface { - AddGetDuration(shardID string, success bool, d time.Duration) - IncGetCounter(shardID string, success bool, storageType string) - - AddDeleteDuration(shardID string, success bool, d time.Duration) - IncDeleteCounter(shardID string, success bool, storageType string) - - AddPutDuration(shardID string, success bool, d time.Duration) - IncPutCounter(shardID string, success bool, storageType string) + AddGetDuration(shardID string, success bool, d time.Duration, storageType string) + AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) + AddPutDuration(shardID string, success bool, d time.Duration, storageType string) IncActualCount(shardID string, storageType string) DecActualCount(shardID string, storageType string) @@ -42,14 +37,9 @@ type WriteCacheMetrics interface { } type writeCacheMetrics struct { - getDuration metric[*prometheus.HistogramVec] - getCounter metric[*prometheus.CounterVec] - - putDuration metric[*prometheus.HistogramVec] - putCounter metric[*prometheus.CounterVec] - + getDuration metric[*prometheus.HistogramVec] + putDuration metric[*prometheus.HistogramVec] deleteDuration metric[*prometheus.HistogramVec] - deleteCounter metric[*prometheus.CounterVec] flushCounter metric[*prometheus.CounterVec] evictCounter metric[*prometheus.CounterVec] @@ -66,11 +56,8 @@ type writeCacheMetrics struct { func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ getDuration: newWCMethodDurationCounter("get"), - getCounter: newWCMethodCounterVec("get"), putDuration: newWCMethodDurationCounter("put"), - putCounter: newWCMethodCounterVec("put"), deleteDuration: newWCMethodDurationCounter("delete"), - deleteCounter: newWCMethodCounterVec("delete"), flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), @@ -81,28 +68,16 @@ func newWriteCacheMetrics() *writeCacheMetrics { } } -func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.getDuration.value, shardID, success, d) +func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.getDuration.value, shardID, success, d, storageType) } -func (m *writeCacheMetrics) IncGetCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.getCounter.value, shardID, success, storageType) +func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.deleteDuration.value, shardID, success, d, storageType) } -func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.deleteDuration.value, shardID, success, d) -} - -func (m *writeCacheMetrics) IncDeleteCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.deleteCounter.value, shardID, success, storageType) -} - -func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.putDuration.value, shardID, success, d) -} - -func (m *writeCacheMetrics) IncPutCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.putCounter.value, shardID, success, storageType) +func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.putDuration.value, shardID, success, d, storageType) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { @@ -187,49 +162,30 @@ func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) func (m *writeCacheMetrics) register() { mustRegister(m.getDuration) - mustRegister(m.getCounter) mustRegister(m.putDuration) - mustRegister(m.putCounter) mustRegister(m.deleteDuration) - mustRegister(m.deleteCounter) mustRegister(m.actualCount) mustRegister(m.estimatedSize) mustRegister(m.flushCounter) mustRegister(m.evictCounter) } -func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration) { +func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { m.With( prometheus.Labels{ wcShardID: shardID, wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, }, ).Observe(float64(d)) } -func incWriteCacheCounter(m *prometheus.CounterVec, shardID string, success bool, storageType string) { - m.With(prometheus.Labels{ - wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), - wcStorage: storageType, - }).Inc() -} - func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { return newHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess}) -} - -func newWCMethodCounterVec(method string) metric[*prometheus.CounterVec] { - return newCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_req_count", method), - Help: fmt.Sprintf("The number of %s requests processed", method), }, []string{wcShardID, wcSuccess, wcStorage}) } -- 2.45.2 From 9c54a2410111087022a5e5ab9d268d0ab85aaeae Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 22 May 2023 09:59:38 +0300 Subject: [PATCH 067/233] [#364] Fix trailing whitespace Signed-off-by: Anton Nikiforov --- docs/evacuation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/evacuation.md b/docs/evacuation.md index 7c8beb790..5befad164 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -85,8 +85,8 @@ Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evac ### Start evacuation and await it completes without progress notifications ```bash frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await --no-progress -Enter password > +Enter password > Shard evacuation has been successfully started. Shard evacuation has been completed. Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. -``` \ No newline at end of file +``` -- 2.45.2 From 802168c0c66b532409e4964cf032fd4e6528006a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 24 May 2023 14:09:11 +0300 Subject: [PATCH 068/233] [#364] node: Stop flushing big object when termination signal received Signed-off-by: Anton Nikiforov --- .../blobstor/blobovniczatree/iterate.go | 3 ++- .../blobstor/common/storage.go | 2 +- .../blobstor/fstree/fstree.go | 13 +++++++++---- .../blobstor/internal/blobstortest/iterate.go | 6 +++--- pkg/local_object_storage/blobstor/iterate.go | 9 +++++---- pkg/local_object_storage/blobstor/iterate_test.go | 2 +- .../blobstor/memstore/memstore.go | 2 +- pkg/local_object_storage/blobstor/perf_test.go | 2 +- .../blobstor/teststore/teststore.go | 4 ++-- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/writecache/flush.go | 15 +++++++++++++-- 11 files changed, 39 insertions(+), 21 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 9918801b9..ad933da0b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" @@ -11,7 +12,7 @@ import ( ) // Iterate iterates over all objects in b. -func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error) { +func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common.IterateRes, error) { return common.IterateRes{}, b.iterateBlobovniczas(prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 801d32c1e..c5d187f30 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -25,5 +25,5 @@ type Storage interface { Exists(context.Context, ExistsPrm) (ExistsRes, error) Put(context.Context, PutPrm) (PutRes, error) Delete(context.Context, DeletePrm) (DeleteRes, error) - Iterate(IteratePrm) (IterateRes, error) + Iterate(context.Context, IteratePrm) (IterateRes, error) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 75f63193a..b0879f68e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -100,11 +100,11 @@ func addressFromString(s string) (oid.Address, error) { } // Iterate iterates over all stored objects. -func (t *FSTree) Iterate(prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, t.iterate(0, []string{t.RootPath}, prm) +func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + return common.IterateRes{}, t.iterate(ctx, 0, []string{t.RootPath}, prm) } -func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) error { +func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { curName := strings.Join(curPath[1:], "") des, err := os.ReadDir(filepath.Join(curPath...)) if err != nil { @@ -119,10 +119,15 @@ func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) curPath = append(curPath, "") for i := range des { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } curPath[l] = des[i].Name() if !isLast && des[i].IsDir() { - err := t.iterate(depth+1, curPath, prm) + err := t.iterate(ctx, depth+1, curPath, prm) if err != nil { // Must be error from handler in case errors are ignored. // Need to report. diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 83ada9607..34622c857 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -49,7 +49,7 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -72,7 +72,7 @@ func runTestLazyHandler(t *testing.T, s common.Storage, objects []objectDesc) { return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -107,7 +107,7 @@ func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []object return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.Equal(t, err, logicErr) require.Equal(t, len(objects)/2, len(seen)) for i := range objects { diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 2c37ee776..4e52f0abf 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,12 +17,12 @@ import ( // did not allow to completely iterate over the storage. // // If handler returns an error, method wraps and returns it immediately. -func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { +func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { b.modeMtx.RLock() defer b.modeMtx.RUnlock() for i := range b.storage { - _, err := b.storage[i].Storage.Iterate(prm) + _, err := b.storage[i].Storage.Iterate(ctx, prm) if err != nil && !prm.IgnoreErrors { return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } @@ -31,7 +32,7 @@ func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { // IterateBinaryObjects is a helper function which iterates over BlobStor and passes binary objects to f. // Errors related to object reading and unmarshaling are logged and skipped. -func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { +func IterateBinaryObjects(ctx context.Context, blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { var prm common.IteratePrm prm.Handler = func(elem common.IterationElement) error { @@ -45,7 +46,7 @@ func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, d return nil } - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 6488ff5fc..c35869655 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -68,7 +68,7 @@ func TestIterateObjects(t *testing.T) { require.NoError(t, err) } - err := IterateBinaryObjects(blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err := IterateBinaryObjects(context.Background(), blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { v, ok := mObjs[string(data)] require.True(t, ok) diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index e435cfef4..b6cca2551 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -126,7 +126,7 @@ func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.D return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } -func (s *memstoreImpl) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() for k, v := range s.objs { diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index f21982530..5245146cb 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -207,7 +207,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { // Benchmark iterate cnt := 0 b.ResetTimer() - _, err := st.Iterate(common.IteratePrm{ + _, err := st.Iterate(context.Background(), common.IteratePrm{ Handler: func(elem common.IterationElement) error { cnt++ return nil diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 24d742fda..c7179eb45 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -202,14 +202,14 @@ func (s *TestStore) Delete(ctx context.Context, req common.DeletePrm) (common.De } } -func (s *TestStore) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Iterate != nil: return s.overrides.Iterate(req) case s.st != nil: - return s.st.Iterate(req) + return s.st.Iterate(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Iterate(%+v)", req)) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index f885451af..e8e2bd4d7 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -170,7 +170,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { obj := objectSDK.New() - err = blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err = blobstor.IterateBinaryObjects(ctx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { if err := obj.Unmarshal(data); err != nil { s.log.Warn(logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 28409f609..767435ebf 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -35,6 +35,17 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ch := c.closeCh + c.wg.Add(1) + go func() { + <-ch + cancel() + c.wg.Done() + }() + for i := 0; i < c.workersCount; i++ { c.wg.Add(1) go c.workerFlushSmall() @@ -42,7 +53,7 @@ func (c *cache) runFlushLoop() { c.wg.Add(1) go func() { - c.workerFlushBig(context.TODO()) + c.workerFlushBig(ctx) c.wg.Done() }() @@ -211,7 +222,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return nil } - _, err := c.fsTree.Iterate(prm) + _, err := c.fsTree.Iterate(ctx, prm) return err } -- 2.45.2 From 365a7ca0f424f85a76b91d4878812bdf34b16729 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 29 May 2023 09:35:08 +0300 Subject: [PATCH 069/233] [#366] node: Stop GC once termination signal received Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/gc.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 78c41cb64..34a48d44a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -76,7 +76,7 @@ type gc struct { workerPool util.WorkerPool - remover func() + remover func(context.Context) eventChan chan Event mEventHandler map[eventType]*eventHandlers @@ -115,7 +115,7 @@ func (gc *gc) init(ctx context.Context) { } gc.wg.Add(2) - go gc.tickRemover() + go gc.tickRemover(ctx) go gc.listenEvents(ctx) } @@ -160,7 +160,7 @@ func (gc *gc) listenEvents(ctx context.Context) { } } -func (gc *gc) tickRemover() { +func (gc *gc) tickRemover(ctx context.Context) { defer gc.wg.Done() timer := time.NewTimer(gc.removerInterval) @@ -178,7 +178,7 @@ func (gc *gc) tickRemover() { gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: - gc.remover() + gc.remover(ctx) timer.Reset(gc.removerInterval) } } @@ -196,8 +196,8 @@ func (gc *gc) stop() { // iterates over metabase and deletes objects // with GC-marked graves. // Does nothing if shard is in "read-only" mode. -func (s *Shard) removeGarbage() { - ctx, cancel := context.WithCancel(context.Background()) +func (s *Shard) removeGarbage(pctx context.Context) { + ctx, cancel := context.WithCancel(pctx) defer cancel() s.gcCancel.Store(cancel) -- 2.45.2 From a3e30062dfa3446e3772ac4ba32bbdc76a200195 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 22 May 2023 13:00:24 +0300 Subject: [PATCH 070/233] [#325] node: Introduce unsafe_disable param to disable policer Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-node/config/policer/config.go | 6 ++++++ cmd/frostfs-node/object.go | 5 +++++ internal/logs/logs.go | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config/policer/config.go b/cmd/frostfs-node/config/policer/config.go index 51e55c341..487e42be8 100644 --- a/cmd/frostfs-node/config/policer/config.go +++ b/cmd/frostfs-node/config/policer/config.go @@ -25,3 +25,9 @@ func HeadTimeout(c *config.Config) time.Duration { return HeadTimeoutDefault } + +// UnsafeDisable returns the value of "unsafe_disable" config parameter +// from "policer" section. +func UnsafeDisable(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "unsafe_disable") +} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1ce330cb5..7a409da03 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -204,6 +204,11 @@ func initObjectService(c *cfg) { } func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { + if policerconfig.UnsafeDisable(c.appCfg) { + c.log.Warn(logs.FrostFSNodePolicerIsDisabled) + return + } + ls := c.cfgObject.cfgLocalStorage.localStorage pol := policer.New( diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a862e745a..9b800adfc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -477,7 +477,8 @@ const ( FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go - CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSNodePolicerIsDisabled = "policer is disabled" + CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" -- 2.45.2 From 8dcd06c5871f98581251d5db666af6970b7ce0a6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Sun, 28 May 2023 22:37:37 +0300 Subject: [PATCH 071/233] [#394] node: Use `Context` in `Blobovniczas.Iterate()` Signed-off-by: Anton Nikiforov --- cmd/frostfs-lens/internal/blobovnicza/list.go | 3 +- .../blobovnicza/iterate.go | 12 ++++++-- .../blobovnicza/iterate_test.go | 9 +++--- .../blobstor/blobovniczatree/control.go | 3 +- .../blobstor/blobovniczatree/delete.go | 2 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 2 +- .../blobstor/blobovniczatree/get_range.go | 2 +- .../blobstor/blobovniczatree/iterate.go | 29 ++++++++++++------- .../blobstor/blobovniczatree/put.go | 2 +- 10 files changed, 41 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-lens/internal/blobovnicza/list.go b/cmd/frostfs-lens/internal/blobovnicza/list.go index 67242a7d1..d327dbc41 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/list.go +++ b/cmd/frostfs-lens/internal/blobovnicza/list.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "fmt" "io" @@ -33,6 +34,6 @@ func listFunc(cmd *cobra.Command, _ []string) { blz := openBlobovnicza(cmd) defer blz.Close() - err := blobovnicza.IterateAddresses(blz, wAddr) + err := blobovnicza.IterateAddresses(context.Background(), blz, wAddr) common.ExitOnErr(cmd, common.Errf("blobovnicza iterator failure: %w", err)) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 1adfacbc0..8f8357053 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "fmt" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -117,12 +118,17 @@ type IterateRes struct { // Returns handler's errors directly. Returns nil after iterating finish. // // Handler should not retain object data. Handler must not be nil. -func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { +func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, error) { var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error { return buck.ForEach(func(k, v []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if prm.decodeAddresses { if err := addressFromKey(&elem.addr, k); err != nil { if prm.ignoreErrors { @@ -147,7 +153,7 @@ func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { } // IterateAddresses is a helper function which iterates over Blobovnicza and passes addresses of the objects to f. -func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { +func IterateAddresses(ctx context.Context, blz *Blobovnicza, f func(oid.Address) error) error { var prm IteratePrm prm.DecodeAddresses() @@ -157,7 +163,7 @@ func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { return f(elem.Address()) }) - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 6ecb20c77..505685ced 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "errors" "path/filepath" "testing" @@ -33,22 +34,22 @@ func TestBlobovniczaIterate(t *testing.T) { return nil } - _, err = b.Iterate(IteratePrm{handler: inc}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc}) require.NoError(t, err) require.ElementsMatch(t, seen, data) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true}) require.Error(t, err) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) require.NoError(t, err) require.ElementsMatch(t, seen, data[:1]) seen = seen[:0] expectedErr := errors.New("stop iteration") - _, err = b.Iterate(IteratePrm{ + _, err = b.Iterate(context.Background(), IteratePrm{ decodeAddresses: true, handler: func(IterationElement) error { return expectedErr }, ignoreErrors: true, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0240c7a97..0b2135631 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" @@ -26,7 +27,7 @@ func (b *Blobovniczas) Init() error { return nil } - return b.iterateLeaves(func(p string) (bool, error) { + return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { blz, err := b.openBlobovniczaNoCache(p) if err != nil { return true, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index f84d8fbe8..5aa9062a6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -48,7 +48,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) // don't process active blobovnicza of the level twice diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 9d9fd4cba..e7852612b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -40,7 +40,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common gPrm.SetAddress(prm.Address) var found bool - err := b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 0b8ccb64f..8955eb148 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G activeCache := make(map[string]struct{}) - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index d6dfe51bd..fb23a9671 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index ad933da0b..140716690 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -12,8 +12,8 @@ import ( ) // Iterate iterates over all objects in b. -func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, b.iterateBlobovniczas(prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { +func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + return common.IterateRes{}, b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { data, err := b.compression.Decompress(elem.ObjectData()) @@ -40,14 +40,14 @@ func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common }) subPrm.DecodeAddresses() - _, err := blz.Iterate(subPrm) + _, err := blz.Iterate(ctx, subPrm) return err }) } // iterator over all Blobovniczas in unsorted order. Break on f's error return. -func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { - return b.iterateLeaves(func(p string) (bool, error) { +func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { + return b.iterateLeaves(ctx, func(p string) (bool, error) { blz, err := b.openBlobovnicza(p) if err != nil { if ignoreErrors { @@ -63,8 +63,9 @@ func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *bl } // iterator over the paths of Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateSortedLeaves(ctx context.Context, addr *oid.Address, f func(string) (bool, error)) error { _, err := b.iterateSorted( + ctx, addr, make([]string, 0, b.blzShallowDepth), b.blzShallowDepth, @@ -75,13 +76,14 @@ func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bo } // iterator over directories with Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateDeepest(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { depth := b.blzShallowDepth if depth > 0 { depth-- } _, err := b.iterateSorted( + ctx, &addr, make([]string, 0, depth), depth, @@ -92,7 +94,7 @@ func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, er } // iterator over particular level of directories. -func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { +func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { indices := indexSlice(b.blzShallowWidth) hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) @@ -100,6 +102,11 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe exec := uint64(len(curPath)) == execDepth for i := range indices { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } if i == 0 { curPath = append(curPath, u64ToHexString(indices[i])) } else { @@ -112,7 +119,7 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } else if stop { return true, nil } - } else if stop, err := b.iterateSorted(addr, curPath, execDepth, f); err != nil { + } else if stop, err := b.iterateSorted(ctx, addr, curPath, execDepth, f); err != nil { return false, err } else if stop { return true, nil @@ -123,8 +130,8 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } // iterator over the paths of Blobovniczas in random order. -func (b *Blobovniczas) iterateLeaves(f func(string) (bool, error)) error { - return b.iterateSortedLeaves(nil, f) +func (b *Blobovniczas) iterateLeaves(ctx context.Context, f func(string) (bool, error)) error { + return b.iterateSortedLeaves(ctx, nil, f) } // makes slice of uint64 values from 0 to number-1. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index ec302d143..4e1d6621f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -45,7 +45,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe PutPrm: putPrm, } - if err := b.iterateDeepest(prm.Address, it.iterate); err != nil { + if err := b.iterateDeepest(ctx, prm.Address, it.iterate); err != nil { return common.PutRes{}, err } else if it.ID == nil { if it.AllFull { -- 2.45.2 From ebcc8afbeeeb97c66cdfefab3cde227081b6c59e Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 26 May 2023 13:24:41 +0300 Subject: [PATCH 072/233] [#374] Add inner-ring event metrics Signed-off-by: Alejandro Lopez --- pkg/innerring/initialization.go | 6 +++ pkg/innerring/metrics/metrics.go | 15 ++++++ pkg/innerring/processors/alphabet/handlers.go | 3 +- .../processors/alphabet/process_emit.go | 16 ++++--- .../processors/alphabet/processor.go | 9 ++++ pkg/innerring/processors/balance/handlers.go | 5 +- .../processors/balance/process_assets.go | 7 ++- pkg/innerring/processors/balance/processor.go | 9 ++++ .../processors/container/handlers.go | 13 +++-- .../processors/container/process_container.go | 47 +++++++++---------- .../processors/container/process_eacl.go | 23 +++++---- .../processors/container/processor.go | 9 ++++ pkg/innerring/processors/frostfs/handlers.go | 25 +++++++--- .../processors/frostfs/process_assets.go | 30 +++++++----- .../processors/frostfs/process_bind.go | 14 +++--- .../processors/frostfs/process_config.go | 7 ++- pkg/innerring/processors/frostfs/processor.go | 9 ++++ .../processors/governance/handlers.go | 5 +- .../processors/governance/process_update.go | 14 +++--- .../processors/governance/processor.go | 11 ++++- pkg/innerring/processors/netmap/handlers.go | 19 ++++---- .../processors/netmap/process_cleanup.go | 7 ++- .../processors/netmap/process_epoch.go | 13 +++-- .../processors/netmap/process_peers.go | 22 +++++---- pkg/innerring/processors/netmap/processor.go | 9 ++++ pkg/innerring/processors/util.go | 16 +++++++ pkg/metrics/innerring.go | 37 ++++++++++++--- 27 files changed, 287 insertions(+), 113 deletions(-) create mode 100644 pkg/innerring/metrics/metrics.go create mode 100644 pkg/innerring/processors/util.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 8e5aef951..c49d22509 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -52,6 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -162,6 +163,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli // create governance processor governanceProcessor, err := governance.New(&governance.Params{ Log: s.log, + Metrics: s.metrics, FrostFSClient: frostfsCli, NetmapClient: s.netmapClient, AlphabetState: s, @@ -231,6 +233,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -255,6 +258,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, @@ -273,6 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -293,6 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, diff --git a/pkg/innerring/metrics/metrics.go b/pkg/innerring/metrics/metrics.go new file mode 100644 index 000000000..002f3afe1 --- /dev/null +++ b/pkg/innerring/metrics/metrics.go @@ -0,0 +1,15 @@ +package metrics + +import "time" + +type Register interface { + SetEpoch(epoch uint64) + SetHealth(s int32) + AddEvent(d time.Duration, typ string, success bool) +} + +type DefaultRegister struct{} + +func (DefaultRegister) SetEpoch(uint64) {} +func (DefaultRegister) SetHealth(int32) {} +func (DefaultRegister) AddEvent(time.Duration, string, bool) {} diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index c0668a4f9..9de075f17 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -2,6 +2,7 @@ package alphabet import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "go.uber.org/zap" @@ -13,7 +14,7 @@ func (ap *Processor) HandleGasEmission(ev event.Event) { // send event to the worker pool - err := ap.pool.Submit(func() { ap.processEmit() }) + err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", ap.processEmit) if err != nil { // there system can be moved into controlled degradation stage ap.log.Warn(logs.AlphabetAlphabetProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 7a268ac52..8a2336011 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -13,12 +13,12 @@ import ( const emitMethod = "emit" -func (ap *Processor) processEmit() { +func (ap *Processor) processEmit() bool { index := ap.irList.AlphabetIndex() if index < 0 { ap.log.Info(logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) - return + return true } contract, ok := ap.alphabetContracts.GetByIndex(index) @@ -26,7 +26,7 @@ func (ap *Processor) processEmit() { ap.log.Debug(logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) - return + return false } // there is no signature collecting, so we don't need extra fee @@ -34,13 +34,13 @@ func (ap *Processor) processEmit() { if err != nil { ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) - return + return false } if ap.storageEmission == 0 { ap.log.Info(logs.AlphabetStorageNodeEmissionIsOff) - return + return true } networkMap, err := ap.netmapClient.NetMap() @@ -48,7 +48,7 @@ func (ap *Processor) processEmit() { ap.log.Warn(logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) - return + return false } nmNodes := networkMap.Nodes() @@ -63,7 +63,7 @@ func (ap *Processor) processEmit() { zap.Int("extra_wallets", extraLen)) if nmLen+extraLen == 0 { - return + return true } gasPerNode := fixedn.Fixed8(ap.storageEmission / uint64(nmLen+extraLen)) @@ -71,6 +71,8 @@ func (ap *Processor) processEmit() { ap.transferGasToNetmapNodes(nmNodes, gasPerNode) ap.transferGasToExtraNodes(pw, gasPerNode) + + return true } func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index cd9088e03..972f84281 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -50,6 +51,7 @@ type ( // protects parsedWallets from concurrent change pwLock *sync.RWMutex log *logger.Logger + metrics metrics.Register pool *ants.Pool alphabetContracts Contracts netmapClient netmapClient @@ -62,6 +64,7 @@ type ( Params struct { ParsedWallets []util.Uint160 Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetContracts Contracts NetmapClient netmapClient @@ -89,10 +92,16 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ parsedWallets: p.ParsedWallets, pwLock: new(sync.RWMutex), log: p.Log, + metrics: metricsRegister, pool: pool, alphabetContracts: p.AlphabetContracts, netmapClient: p.NetmapClient, diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index e325da1f9..e39f3abbd 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" "go.uber.org/zap" @@ -17,7 +18,9 @@ func (bp *Processor) handleLock(ev event.Event) { // send an event to the worker pool - err := bp.pool.Submit(func() { bp.processLock(&lock) }) + err := processors.SubmitEvent(bp.pool, bp.metrics, "lock", func() bool { + return bp.processLock(&lock) + }) if err != nil { // there system can be moved into controlled degradation stage bp.log.Warn(logs.BalanceBalanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index 3f86a3cb7..1d94fa454 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -9,10 +9,10 @@ import ( // Process lock event by invoking Cheque method in main net to send assets // back to the withdraw issuer. -func (bp *Processor) processLock(lock *balanceEvent.Lock) { +func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { if !bp.alphabetState.IsAlphabet() { bp.log.Info(logs.BalanceNonAlphabetModeIgnoreBalanceLock) - return + return true } prm := frostfsContract.ChequePrm{} @@ -26,5 +26,8 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) { err := bp.frostfsClient.Cheque(prm) if err != nil { bp.log.Error(logs.BalanceCantSendLockAssetTx, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 356754cfb..5cc849b5c 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" @@ -32,6 +33,7 @@ type ( // Processor of events produced by balance contract in the morphchain. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient balanceSC util.Uint160 @@ -42,6 +44,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSClient FrostFSClient BalanceSC util.Uint160 @@ -72,8 +75,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/balance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, balanceSC: p.BalanceSC, diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 2ab1147c8..3ec10b889 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "github.com/mr-tron/base58" @@ -20,7 +21,9 @@ func (cp *Processor) handlePut(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerPut(put) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_put", func() bool { + return cp.processContainerPut(put) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -36,7 +39,9 @@ func (cp *Processor) handleDelete(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerDelete(del) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_delete", func() bool { + return cp.processContainerDelete(del) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -53,8 +58,8 @@ func (cp *Processor) handleSetEACL(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { - cp.processSetEACL(e) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_set_eacl", func() bool { + return cp.processSetEACL(e) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 3bee1c4d5..33ef90034 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -31,10 +31,10 @@ type putContainerContext struct { // Process a new container from the user by checking the container sanity // and sending approve tx back to the morph. -func (cp *Processor) processContainerPut(put putEvent) { +func (cp *Processor) processContainerPut(put putEvent) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerPut) - return + return true } ctx := &putContainerContext{ @@ -47,10 +47,17 @@ func (cp *Processor) processContainerPut(put putEvent) { zap.String("error", err.Error()), ) - return + return false } - cp.approvePutContainer(ctx) + if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApprovePutContainer, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { @@ -89,20 +96,12 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return nil } -func (cp *Processor) approvePutContainer(ctx *putContainerContext) { - if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApprovePutContainer, - zap.String("error", err.Error()), - ) - } -} - // Process delete container operation from the user by checking container sanity // and sending approve tx back to morph. -func (cp *Processor) processContainerDelete(e containerEvent.Delete) { +func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) - return + return true } err := cp.checkDeleteContainer(e) @@ -111,10 +110,18 @@ func (cp *Processor) processContainerDelete(e containerEvent.Delete) { zap.String("error", err.Error()), ) - return + return false } - cp.approveDeleteContainer(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, + zap.String("error", err.Error()), + ) + + return false + } + + return true } func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { @@ -149,14 +156,6 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { return nil } -func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { - if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, - zap.String("error", err.Error()), - ) - } -} - func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { // fetch domain info ctx.d = containerSDK.ReadDomain(cnr) diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index 43a59e223..8ab0d5c39 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -12,10 +12,10 @@ import ( "go.uber.org/zap" ) -func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { +func (cp *Processor) processSetEACL(e containerEvent.SetEACL) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) - return + return true } err := cp.checkSetEACL(e) @@ -24,10 +24,17 @@ func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { zap.String("error", err.Error()), ) - return + return false } - cp.approveSetEACL(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveSetEACL, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { @@ -73,11 +80,3 @@ func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { return nil } - -func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { - if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveSetEACL, - zap.String("error", err.Error()), - ) - } -} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index ec82ace70..fd5348c6f 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -40,6 +41,7 @@ type ( // Processor of events produced by container contract in the sidechain. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool alphabetState AlphabetState cnrClient ContClient // notary must be enabled @@ -51,6 +53,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetState AlphabetState ContainerClient ContClient @@ -102,8 +105,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/container: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, alphabetState: p.AlphabetState, cnrClient: p.ContainerClient, diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 574cf057a..ab53d5c48 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -18,7 +19,9 @@ func (np *Processor) handleDeposit(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processDeposit(deposit) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_deposit", func() bool { + return np.processDeposit(deposit) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -34,7 +37,9 @@ func (np *Processor) handleWithdraw(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processWithdraw(withdraw) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_withdraw", func() bool { + return np.processWithdraw(withdraw) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -50,7 +55,9 @@ func (np *Processor) handleCheque(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processCheque(cheque) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_cheque", func() bool { + return np.processCheque(cheque) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -67,7 +74,9 @@ func (np *Processor) handleConfig(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processConfig(cfg) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_config", func() bool { + return np.processConfig(cfg) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -83,7 +92,9 @@ func (np *Processor) handleBind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, true) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_bind", func() bool { + return np.processBind(e, true) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -99,7 +110,9 @@ func (np *Processor) handleUnbind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, false) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_unbind", func() bool { + return np.processBind(e, false) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index cfbf21b08..327a4a3aa 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -15,10 +15,10 @@ const ( // Process deposit event by invoking a balance contract and sending native // gas in the sidechain. -func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { +func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) - return + return true } prm := balance.MintPrm{} @@ -49,7 +49,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) - return + return false } // get gas balance of the node @@ -57,7 +57,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { balance, err := np.morphClient.GasBalance() if err != nil { np.log.Error(logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) - return + return false } if balance < np.gasBalanceThreshold { @@ -65,7 +65,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) - return + return false } err = np.morphClient.TransferGas(receiver, np.mintEmitValue) @@ -73,24 +73,26 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { np.log.Error(logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) - return + return false } np.mintEmitCache.Add(receiver.String(), curEpoch) + + return true } // Process withdraw event by locking assets in the balance account. -func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { +func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) - return + return true } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { np.log.Error(logs.FrostFSCantCreateLockAccount, zap.Error(err)) - return + return false } curEpoch := np.epochState.EpochCounter() @@ -106,15 +108,18 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { err = np.balanceClient.Lock(prm) if err != nil { np.log.Error(logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) + return false } + + return true } // Process cheque event by transferring assets from the lock account back to // the reserve account. -func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { +func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) - return + return true } prm := balance.BurnPrm{} @@ -126,5 +131,8 @@ func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { err := np.balanceClient.Burn(prm) if err != nil { np.log.Error(logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go index a9b523a77..50c6bf5f5 100644 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ b/pkg/innerring/processors/frostfs/process_bind.go @@ -18,10 +18,10 @@ type bindCommon interface { TxHash() util.Uint256 } -func (np *Processor) processBind(e bindCommon, bind bool) { +func (np *Processor) processBind(e bindCommon, bind bool) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) - return + return true } c := &bindCommonContext{ @@ -36,10 +36,10 @@ func (np *Processor) processBind(e bindCommon, bind bool) { zap.String("error", err.Error()), ) - return + return false } - np.approveBindCommon(c) + return np.approveBindCommon(c) == nil } type bindCommonContext struct { @@ -70,7 +70,7 @@ func (np *Processor) checkBindCommon(e *bindCommonContext) error { return nil } -func (np *Processor) approveBindCommon(e *bindCommonContext) { +func (np *Processor) approveBindCommon(e *bindCommonContext) error { // calculate wallet address scriptHash := e.User() @@ -80,7 +80,7 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { zap.String("error", err.Error()), ) - return + return err } var id user.ID @@ -104,4 +104,6 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { np.log.Error(fmt.Sprintf("could not approve %s", typ), zap.String("error", err.Error())) } + + return err } diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index ce2dabfdd..2ae3e6ced 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -9,10 +9,10 @@ import ( // Process config event by setting configuration value from the mainchain in // the sidechain. -func (np *Processor) processConfig(config frostfsEvent.Config) { +func (np *Processor) processConfig(config frostfsEvent.Config) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) - return + return true } prm := nmClient.SetConfigPrm{} @@ -25,5 +25,8 @@ func (np *Processor) processConfig(config frostfsEvent.Config) { err := np.netmapClient.SetConfig(prm) if err != nil { np.log.Error(logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 2af15a814..e6ee78837 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -6,6 +6,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -58,6 +59,7 @@ type ( // Processor of events produced by frostfs contract in main net. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsContract util.Uint160 balanceClient BalanceClient @@ -77,6 +79,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSContract util.Uint160 FrostFSIDClient IDClient @@ -129,8 +132,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create LRU cache for gas emission: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsContract: p.FrostFSContract, balanceClient: p.BalanceClient, diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index 727acc21a..fd7f539c3 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -2,6 +2,7 @@ package governance import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" "github.com/nspcc-dev/neo-go/pkg/core/native" @@ -35,7 +36,9 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { // send event to the worker pool - err := gp.pool.Submit(func() { gp.processAlphabetSync(hash) }) + err := processors.SubmitEvent(gp.pool, gp.metrics, "alphabet_sync", func() bool { + return gp.processAlphabetSync(hash) + }) if err != nil { // there system can be moved into controlled degradation stage gp.log.Warn(logs.GovernanceGovernanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 3eae676d4..50ba58e77 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -18,36 +18,36 @@ const ( alphabetUpdateIDPrefix = "AlphabetUpdate" ) -func (gp *Processor) processAlphabetSync(txHash util.Uint256) { +func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { if !gp.alphabetState.IsAlphabet() { gp.log.Info(logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) - return + return true } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) - return + return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) - return + return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { gp.log.Error(logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) - return + return false } if newAlphabet == nil { gp.log.Info(logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) - return + return true } gp.log.Info(logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, @@ -77,6 +77,8 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) { gp.updateFrostFSContractInMainnet(newAlphabet) gp.log.Info(logs.GovernanceFinishedAlphabetListUpdate) + + return true } func prettyKeys(keys keys.PublicKeys) string { diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 07b5b5cef..fa267eade 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -75,6 +76,7 @@ type ( // Processor of events related to governance in the network. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient netmapClient NetmapClient @@ -92,7 +94,8 @@ type ( // Params of the processor constructor. Params struct { - Log *logger.Logger + Log *logger.Logger + Metrics metrics.Register AlphabetState AlphabetState EpochState EpochState @@ -130,11 +133,17 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/governance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + // result is cached by neo-go, so we can pre-calc it designate := p.MainnetClient.GetDesignateHash() return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, netmapClient: p.NetmapClient, diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 6adeac562..c6053e281 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -16,7 +17,7 @@ func (np *Processor) HandleNewEpochTick(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { np.processNewEpochTick() }) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", np.processNewEpochTick) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, @@ -32,8 +33,8 @@ func (np *Processor) handleNewEpoch(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processNewEpoch(epochEvent) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch", func() bool { + return np.processNewEpoch(epochEvent) }) if err != nil { // there system can be moved into controlled degradation stage @@ -51,8 +52,8 @@ func (np *Processor) handleAddPeer(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processAddPeer(newPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_add_peer", func() bool { + return np.processAddPeer(newPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -69,8 +70,8 @@ func (np *Processor) handleUpdateState(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { - np.processUpdatePeer(updPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_update_peer", func() bool { + return np.processUpdatePeer(updPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -91,8 +92,8 @@ func (np *Processor) handleCleanupTick(ev event.Event) { np.log.Info(logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool - err := np.pool.Submit(func() { - np.processNetmapCleanupTick(cleanup) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { + return np.processNetmapCleanupTick(cleanup) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 287844a6a..170c39e2c 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -7,11 +7,11 @@ import ( "go.uber.org/zap" ) -func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { +func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) - return + return true } err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { @@ -47,5 +47,8 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { if err != nil { np.log.Warn(logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index b655db9aa..01bfbae67 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -10,7 +10,7 @@ import ( // Process new epoch notification by setting global epoch value and resetting // local epoch timer. -func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { +func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { epoch := ev.EpochNumber() epochDuration, err := np.netmapClient.EpochDuration() @@ -41,7 +41,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.log.Warn(logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) - return + return false } prm := cntClient.StartEstimationPrm{} @@ -63,13 +63,15 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) np.handleNotaryDeposit(ev) + + return true } // Process new epoch tick by invoking new epoch method in network map contract. -func (np *Processor) processNewEpochTick() { +func (np *Processor) processNewEpochTick() bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewEpochTick) - return + return true } nextEpoch := np.epochState.EpochCounter() + 1 @@ -78,5 +80,8 @@ func (np *Processor) processNewEpochTick() { err := np.netmapClient.NewEpoch(nextEpoch, false) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index e4f1a4d6c..96b8c8e97 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -12,10 +12,10 @@ import ( // Process add peer notification by sanity check of new node // local epoch timer. -func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { +func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) - return + return true } // check if notary transaction is valid, see #976 @@ -26,7 +26,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { zap.String("method", "netmap.AddPeer"), zap.String("hash", tx.Hash().StringLE()), zap.Error(err)) - return + return false } // unmarshal node info @@ -34,7 +34,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it np.log.Warn(logs.NetmapCantParseNetworkMapCandidate) - return + return false } // validate and update node info @@ -44,7 +44,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { zap.String("error", err.Error()), ) - return + return false } // sort attributes to make it consistent @@ -81,15 +81,18 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) + return false } } + + return true } // Process update peer notification by sending approval tx to the smart contract. -func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { +func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) - return + return true } // flag node to remove from local view, so it can be re-bootstrapped @@ -105,11 +108,14 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { zap.Error(err), ) - return + return false } } if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 5984cbbe9..6b8a24a62 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -72,6 +73,7 @@ type ( // and new epoch ticker, because it is related to contract. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool epochTimer EpochTimerReseter epochState EpochState @@ -93,6 +95,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int NetmapClient Client EpochTimer EpochTimerReseter @@ -145,8 +148,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/netmap: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, epochTimer: p.EpochTimer, epochState: p.EpochState, diff --git a/pkg/innerring/processors/util.go b/pkg/innerring/processors/util.go new file mode 100644 index 000000000..364ffe25e --- /dev/null +++ b/pkg/innerring/processors/util.go @@ -0,0 +1,16 @@ +package processors + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" + "github.com/panjf2000/ants/v2" +) + +func SubmitEvent(pool *ants.Pool, metrics metrics.Register, eventLabel string, eventProcessor func() bool) error { + return pool.Submit(func() { + start := time.Now() + success := eventProcessor() + metrics.AddEvent(time.Since(start), eventLabel, success) + }) +} diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 79db424b8..bff9184ec 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -1,13 +1,23 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "strconv" + "time" -const innerRingSubsystem = "ir" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + innerRingSubsystem = "ir" + innerRingLabelSuccess = "success" + innerRingLabelType = "type" +) // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch metric[prometheus.Gauge] - health metric[prometheus.Gauge] + epoch metric[prometheus.Gauge] + health metric[prometheus.Gauge] + eventDuration metric[*prometheus.HistogramVec] } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -25,14 +35,22 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { Name: "health", Help: "Current inner-ring node state.", }) + eventDuration = newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "event_duration_seconds", + Help: "Duration of processing of inner-ring events", + }, []string{innerRingLabelType, innerRingLabelSuccess}) ) mustRegister(epoch) mustRegister(health) + mustRegister(eventDuration) return &InnerRingServiceMetrics{ - epoch: epoch, - health: health, + epoch: epoch, + health: health, + eventDuration: eventDuration, } } @@ -45,3 +63,10 @@ func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { func (m InnerRingServiceMetrics) SetHealth(s int32) { m.health.value.Set(float64(s)) } + +func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { + m.eventDuration.value.With(prometheus.Labels{ + innerRingLabelType: typ, + innerRingLabelSuccess: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} -- 2.45.2 From f64322576a86c5b35489bf1adfca73e03aaba4ef Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 May 2023 18:44:46 +0300 Subject: [PATCH 073/233] [#402] cli: Add estimated evacuation time left Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 45152fa0a..4eb6505cf 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -212,9 +212,28 @@ func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusRespo appendError(sb, resp) appendStartedAt(sb, resp) appendDuration(sb, resp) + appendEstimation(sb, resp) cmd.Println(sb.String()) } +func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || + resp.GetBody().GetDuration() == nil || + resp.GetBody().GetTotal() == 0 || + resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed() == 0 { + return + } + + durationSeconds := float64(resp.GetBody().GetDuration().GetSeconds()) + evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed()) + avgObjEvacuationTimeSeconds := durationSeconds / evacuated + objectsLeft := float64(resp.GetBody().GetTotal()) - evacuated + leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft + leftMinutes := int(leftSeconds / 60) + + sb.WriteString(fmt.Sprintf(" Estimated time left: %d minutes.", leftMinutes)) +} + func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if resp.GetBody().GetDuration() != nil { duration := time.Second * time.Duration(resp.GetBody().GetDuration().GetSeconds()) -- 2.45.2 From f934abed8f2a1e6b6870ef8cc190fad987cc1845 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 May 2023 10:11:08 +0300 Subject: [PATCH 074/233] [#402] doc: Update evacuation docs Add estimated time left output to examples. Signed-off-by: Dmitrii Stepanov --- docs/evacuation.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/evacuation.md b/docs/evacuation.md index 5befad164..9bfa0e214 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -39,15 +39,15 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. Estimated time left: 2 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:05. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:01:05. Estimated time left: 1 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:13. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:02:13. ``` ### Stop running evacuation process @@ -58,7 +58,7 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. Estimated time left: 0 minutes. frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > @@ -75,9 +75,9 @@ frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 - Enter password > Shard evacuation has been successfully started. Progress will be reported every 5 seconds. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. Estimated time left: 0 minutes. Shard evacuation has been completed. Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. ``` -- 2.45.2 From faca8614513a8fab594152b146d89270a0cd4576 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 30 May 2023 10:14:37 +0300 Subject: [PATCH 075/233] [#411] Remove unnecessary pointers for sync objects Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/cache.go | 8 +++----- cmd/frostfs-node/config.go | 3 +-- pkg/innerring/processors/alphabet/processor.go | 3 +-- pkg/innerring/processors/frostfs/processor.go | 3 +-- .../processors/netmap/cleanup_table.go | 3 +-- pkg/local_object_storage/engine/engine.go | 17 +++++++---------- .../engine/evacuate_limiter.go | 2 +- pkg/morph/client/client.go | 4 ++-- pkg/morph/client/constructor.go | 15 ++++++--------- pkg/morph/subscriber/subscriber.go | 3 +-- pkg/morph/timer/block.go | 4 +--- pkg/services/notificator/nats/service.go | 3 +-- pkg/services/object/get/v2/get_forwarder.go | 4 ++-- .../object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/get/v2/util.go | 13 ++++--------- .../object/search/v2/request_forwarder.go | 2 +- pkg/services/object/search/v2/util.go | 6 ++---- .../object_manager/placement/traverser.go | 3 +-- .../session/storage/temporary/storage.go | 3 +-- 20 files changed, 39 insertions(+), 64 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index dfbaf3525..bae38b299 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -25,19 +25,18 @@ type valueWithTime[V any] struct { } type locker struct { - mtx *sync.Mutex + mtx sync.Mutex waiters int // not protected by mtx, must used outer mutex to update concurrently } type keyLocker[K comparable] struct { lockers map[K]*locker - lockersMtx *sync.Mutex + lockersMtx sync.Mutex } func newKeyLocker[K comparable]() *keyLocker[K] { return &keyLocker[K]{ - lockers: make(map[K]*locker), - lockersMtx: &sync.Mutex{}, + lockers: make(map[K]*locker), } } @@ -53,7 +52,6 @@ func (l *keyLocker[K]) LockKey(key K) { } locker := &locker{ - mtx: &sync.Mutex{}, waiters: 1, } locker.mtx.Lock() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 30d457967..00ade7ce2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -327,7 +327,7 @@ type internals struct { log *logger.Logger - wg *sync.WaitGroup + wg sync.WaitGroup workers []worker closers []closer @@ -589,7 +589,6 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { appCfg: appCfg, internalErr: make(chan error), log: log, - wg: new(sync.WaitGroup), apiVersion: version.Current(), healthStatus: &healthStatus, } diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 972f84281..04dde80f7 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -49,7 +49,7 @@ type ( Processor struct { parsedWallets []util.Uint160 // protects parsedWallets from concurrent change - pwLock *sync.RWMutex + pwLock sync.RWMutex log *logger.Logger metrics metrics.Register pool *ants.Pool @@ -99,7 +99,6 @@ func New(p *Params) (*Processor, error) { return &Processor{ parsedWallets: p.ParsedWallets, - pwLock: new(sync.RWMutex), log: p.Log, metrics: metricsRegister, pool: pool, diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index e6ee78837..20f44adcd 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -68,7 +68,7 @@ type ( epochState EpochState alphabetState AlphabetState converter PrecisionConverter - mintEmitLock *sync.Mutex + mintEmitLock sync.Mutex mintEmitCache *lru.Cache[string, uint64] mintEmitThreshold uint64 mintEmitValue fixedn.Fixed8 @@ -148,7 +148,6 @@ func New(p *Params) (*Processor, error) { epochState: p.EpochState, alphabetState: p.AlphabetState, converter: p.Converter, - mintEmitLock: new(sync.Mutex), mintEmitCache: lruCache, mintEmitThreshold: p.MintEmitThreshold, mintEmitValue: p.MintEmitValue, diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index e4024e95f..80117247d 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -9,7 +9,7 @@ import ( type ( cleanupTable struct { - *sync.RWMutex + sync.RWMutex enabled bool threshold uint64 lastAccess map[string]epochStampWithNodeInfo @@ -29,7 +29,6 @@ type ( func newCleanupTable(enabled bool, threshold uint64) cleanupTable { return cleanupTable{ - RWMutex: new(sync.RWMutex), enabled: enabled, threshold: threshold, lastAccess: make(map[string]epochStampWithNodeInfo), diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 19e7b5237..21e863005 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -20,7 +20,7 @@ type StorageEngine struct { removeDuplicatesInProgress atomic.Bool - mtx *sync.RWMutex + mtx sync.RWMutex shards map[string]hashedShard @@ -225,15 +225,12 @@ func New(opts ...Option) *StorageEngine { } return &StorageEngine{ - cfg: c, - mtx: new(sync.RWMutex), - shards: make(map[string]hashedShard), - shardPools: make(map[string]util.WorkerPool), - closeCh: make(chan struct{}), - setModeCh: make(chan setModeRequest), - evacuateLimiter: &evacuationLimiter{ - guard: &sync.RWMutex{}, - }, + cfg: c, + shards: make(map[string]hashedShard), + shardPools: make(map[string]util.WorkerPool), + closeCh: make(chan struct{}), + setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{}, } } diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 425fdc775..62795fa1a 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -113,7 +113,7 @@ type evacuationLimiter struct { eg *errgroup.Group cancel context.CancelFunc - guard *sync.RWMutex + guard sync.RWMutex } func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 6b6e1284b..de29d0c03 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -66,7 +66,7 @@ type Client struct { // switchLock protects endpoints, inactive, and subscription-related fields. // It is taken exclusively during endpoint switch and locked in shared mode // on every normal call. - switchLock *sync.RWMutex + switchLock sync.RWMutex // channel for internal stop closeChan chan struct{} @@ -83,7 +83,7 @@ type Client struct { } type cache struct { - m *sync.RWMutex + m sync.RWMutex nnsHash *util.Uint160 gKey *keys.PublicKey diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 1f2a1eb8d..ff37da728 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -105,13 +104,12 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), - logger: cfg.logger, - acc: acc, - accAddr: accAddr, - cfg: *cfg, - switchLock: &sync.RWMutex{}, - closeChan: make(chan struct{}), + cache: newClientCache(), + logger: cfg.logger, + acc: acc, + accAddr: accAddr, + cfg: *cfg, + closeChan: make(chan struct{}), } cli.endpoints.init(cfg.endpoints) @@ -198,7 +196,6 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto func newClientCache() cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ - m: &sync.RWMutex{}, txHeights: c, } } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 383d58407..608872dec 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -42,7 +42,7 @@ type ( } subscriber struct { - *sync.RWMutex + sync.RWMutex log *logger.Logger client *client.Client @@ -163,7 +163,6 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { } sub := &subscriber{ - RWMutex: new(sync.RWMutex), log: p.Log, client: p.Client, notifyChan: make(chan *state.ContainedNotificationEvent), diff --git a/pkg/morph/timer/block.go b/pkg/morph/timer/block.go index 31c28e2ff..be20d3571 100644 --- a/pkg/morph/timer/block.go +++ b/pkg/morph/timer/block.go @@ -17,7 +17,7 @@ type BlockTickHandler func() type BlockTimer struct { rolledBack bool - mtx *sync.Mutex + mtx sync.Mutex dur BlockMeter @@ -64,7 +64,6 @@ func StaticBlockMeter(d uint32) BlockMeter { // Reset should be called before timer ticking. func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, @@ -80,7 +79,6 @@ func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { // Do not use delta handlers with pulse in this timer. func NewOneTickTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 6a7e80a53..19f0cbde1 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -25,7 +25,7 @@ type Writer struct { js nats.JetStreamContext nc *nats.Conn - m *sync.RWMutex + m sync.RWMutex createdStreams map[string]struct{} opts } @@ -84,7 +84,6 @@ func (n *Writer) Notify(topic string, address oid.Address) error { // New creates new Writer. func New(oo ...Option) *Writer { w := &Writer{ - m: &sync.RWMutex{}, createdStreams: make(map[string]struct{}), opts: opts{ log: &logger.Logger{Logger: zap.L()}, diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index d11f94b26..bc27f69a2 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -23,8 +23,8 @@ import ( ) type getRequestForwarder struct { - OnceResign *sync.Once - OnceHeaderSending *sync.Once + OnceResign sync.Once + OnceHeaderSending sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRequest diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 6c744b23a..1137ba33c 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -23,7 +23,7 @@ import ( ) type getRangeRequestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRangeRequest diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 0c91ec5d8..1b0374aea 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -25,7 +25,7 @@ import ( type headRequestForwarder struct { Request *objectV2.HeadRequest Response *objectV2.HeadResponse - OnceResign *sync.Once + OnceResign sync.Once ObjectAddr oid.Address Key *ecdsa.PrivateKey } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 91e7a96a2..519d9afa0 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "errors" "hash" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" @@ -59,12 +58,10 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre } forwarder := &getRequestForwarder{ - OnceResign: &sync.Once{}, - OnceHeaderSending: &sync.Once{}, - GlobalProgress: 0, - Key: key, - Request: req, - Stream: streamWrapper, + GlobalProgress: 0, + Key: key, + Request: req, + Stream: streamWrapper, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) @@ -115,7 +112,6 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get } forwarder := &getRangeRequestForwarder{ - OnceResign: &sync.Once{}, GlobalProgress: 0, Key: key, Request: req, @@ -254,7 +250,6 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon forwarder := &headRequestForwarder{ Request: req, Response: resp, - OnceResign: &sync.Once{}, ObjectAddr: objAddr, Key: key, } diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go index d8719986f..5a2e9b936 100644 --- a/pkg/services/object/search/v2/request_forwarder.go +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -20,7 +20,7 @@ import ( ) type requestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once Request *objectV2.SearchRequest Key *ecdsa.PrivateKey } diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index 12158a820..cfccaede6 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -51,9 +50,8 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre } forwarder := &requestForwarder{ - OnceResign: &sync.Once{}, - Request: req, - Key: key, + Request: req, + Key: key, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequest)) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index e46240a86..7b9b26a0b 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -29,7 +29,7 @@ type Option func(*cfg) // Traverser represents utility for controlling // traversal of object placement vectors. type Traverser struct { - mtx *sync.RWMutex + mtx sync.RWMutex vectors [][]netmap.NodeInfo @@ -107,7 +107,6 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } return &Traverser{ - mtx: new(sync.RWMutex), rem: rem, vectors: ns, }, nil diff --git a/pkg/services/session/storage/temporary/storage.go b/pkg/services/session/storage/temporary/storage.go index 370499e06..ee93dee71 100644 --- a/pkg/services/session/storage/temporary/storage.go +++ b/pkg/services/session/storage/temporary/storage.go @@ -18,7 +18,7 @@ type key struct { // expiring (removing) session tokens. // Must be created only via calling NewTokenStore. type TokenStore struct { - mtx *sync.RWMutex + mtx sync.RWMutex tokens map[key]*storage.PrivateToken } @@ -28,7 +28,6 @@ type TokenStore struct { // The elements of the instance are stored in the map. func NewTokenStore() *TokenStore { return &TokenStore{ - mtx: new(sync.RWMutex), tokens: make(map[key]*storage.PrivateToken), } } -- 2.45.2 From 3220c4df9fb513d5f6cd3b15e056cc9ea5e8f49e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 May 2023 17:32:13 +0300 Subject: [PATCH 076/233] [#376] metrics: Add GC metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 22 +++++ pkg/local_object_storage/engine/shards.go | 21 +++-- pkg/local_object_storage/shard/delete.go | 12 ++- pkg/local_object_storage/shard/gc.go | 95 ++++++++++++++++--- pkg/local_object_storage/shard/shard.go | 7 ++ pkg/metrics/gc.go | 104 +++++++++++++++++++++ pkg/metrics/node.go | 9 ++ 7 files changed, 244 insertions(+), 26 deletions(-) create mode 100644 pkg/metrics/gc.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 7a11888c5..1be888eae 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -28,6 +28,7 @@ type MetricRegister interface { AddToPayloadCounter(shardID string, size int64) WriteCache() metrics.WriteCacheMetrics + GC() metrics.GCMetrics } func elapsed(addFunc func(d time.Duration)) func() { @@ -37,3 +38,24 @@ func elapsed(addFunc func(d time.Duration)) func() { addFunc(time.Since(t)) } } + +type gcMetrics struct { + storage metrics.GCMetrics + shardID string +} + +func (m *gcMetrics) AddRunDuration(d time.Duration, success bool) { + m.storage.AddRunDuration(m.shardID, d, success) +} + +func (m *gcMetrics) AddDeletedCount(deleted, failed uint64) { + m.storage.AddDeletedCount(m.shardID, deleted, failed) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) { + m.storage.AddExpiredObjectCollectionDuration(m.shardID, d, success, objectType) +} + +func (m *gcMetrics) AddInhumedObjectCount(count uint64, objectType string) { + m.storage.AddInhumedObjectCount(m.shardID, count, objectType) +} diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 6c49c8312..07d22d3fe 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -98,13 +98,20 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { id: id.String(), mw: e.metrics, }, - )) - opts = append(opts, shard.WithExtraWriteCacheOptions(writecache.WithMetrics( - &writeCacheMetrics{ - shardID: id.String(), - metrics: e.metrics.WriteCache(), - }, - ))) + ), + shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }), + ), + shard.WithGCMetrics( + &gcMetrics{ + storage: e.metrics.GC(), + shardID: id.String(), + }, + ), + ) } e.mtx.RUnlock() diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 4eb7ad6af..4843314de 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -21,7 +21,9 @@ type DeletePrm struct { } // DeleteRes groups the resulting values of Delete operation. -type DeleteRes struct{} +type DeleteRes struct { + deleted uint64 +} // SetAddresses is a Delete option to set the addresses of the objects to delete. // @@ -53,10 +55,11 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrDegradedMode } + result := DeleteRes{} for _, addr := range prm.addr { select { case <-ctx.Done(): - return DeleteRes{}, ctx.Err() + return result, ctx.Err() default: } @@ -65,11 +68,12 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { s.deleteFromBlobstorSafe(ctx, addr) if err := s.deleteFromMetabase(ctx, addr); err != nil { - return DeleteRes{}, err // stop on metabase error ? + return result, err // stop on metabase error ? } + result.deleted++ } - return DeleteRes{}, nil + return result, nil } func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 34a48d44a..2580173c2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -67,6 +67,32 @@ type eventHandlers struct { handlers []eventHandler } +type gcRunResult struct { + success bool + deleted uint64 + failedToDelete uint64 +} + +const ( + objectTypeLock = "lock" + objectTypeTombstone = "tombstone" + objectTypeRegular = "regular" +) + +type GCMectrics interface { + AddRunDuration(d time.Duration, success bool) + AddDeletedCount(deleted, failed uint64) + AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) + AddInhumedObjectCount(count uint64, objectType string) +} + +type noopGCMetrics struct{} + +func (m *noopGCMetrics) AddRunDuration(time.Duration, bool) {} +func (m *noopGCMetrics) AddDeletedCount(uint64, uint64) {} +func (m *noopGCMetrics) AddExpiredObjectCollectionDuration(time.Duration, bool, string) {} +func (m *noopGCMetrics) AddInhumedObjectCount(uint64, string) {} + type gc struct { *gcCfg @@ -76,7 +102,7 @@ type gc struct { workerPool util.WorkerPool - remover func(context.Context) + remover func(context.Context) gcRunResult eventChan chan Event mEventHandler map[eventType]*eventHandlers @@ -91,6 +117,8 @@ type gcCfg struct { expiredCollectorWorkersCount int expiredCollectorBatchSize int + + metrics GCMectrics } func defaultGCCfg() gcCfg { @@ -100,6 +128,7 @@ func defaultGCCfg() gcCfg { workerPoolInit: func(int) util.WorkerPool { return nil }, + metrics: &noopGCMetrics{}, } } @@ -178,8 +207,13 @@ func (gc *gc) tickRemover(ctx context.Context) { gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: - gc.remover(ctx) + startedAt := time.Now() + + result := gc.remover(ctx) timer.Reset(gc.removerInterval) + + gc.metrics.AddRunDuration(time.Since(startedAt), result.success) + gc.metrics.AddDeletedCount(result.deleted, result.failedToDelete) } } } @@ -196,7 +230,7 @@ func (gc *gc) stop() { // iterates over metabase and deletes objects // with GC-marked graves. // Does nothing if shard is in "read-only" mode. -func (s *Shard) removeGarbage(pctx context.Context) { +func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { ctx, cancel := context.WithCancel(pctx) defer cancel() @@ -244,6 +278,7 @@ func (s *Shard) removeGarbage(pctx context.Context) { return } else if len(buf) == 0 { + result.success = true return } @@ -251,14 +286,20 @@ func (s *Shard) removeGarbage(pctx context.Context) { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(ctx, deletePrm) + res, err := s.delete(ctx, deletePrm) + + result.deleted = res.deleted + result.failedToDelete = uint64(len(buf)) - res.deleted + result.success = true + if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), ) - - return + result.success = false } + + return } func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { @@ -276,6 +317,13 @@ func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { } func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeRegular) + }() + s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) @@ -286,7 +334,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { if o.Type() != object.TypeTombstone && o.Type() != object.TypeLock { batch = append(batch, o.Address()) @@ -300,8 +348,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -315,7 +363,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) } } @@ -355,6 +403,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -380,6 +429,13 @@ func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error } func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeTombstone) + }() + epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) @@ -413,7 +469,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err := s.metaBase.IterateOverGraveyard(iterPrm) + err = s.metaBase.IterateOverGraveyard(iterPrm) if err != nil { log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() @@ -444,6 +500,13 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeLock) + }() + s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) @@ -455,7 +518,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { if o.Type() == object.TypeLock { batch = append(batch, o.Address()) @@ -469,8 +532,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -484,7 +547,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) } } @@ -553,6 +616,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -598,6 +662,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 65cc1ef55..2123bca1f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -309,6 +309,13 @@ func WithMetricsWriter(v MetricsWriter) Option { } } +// WithGCMetrics returns option to specify storage of the GC metrics. +func WithGCMetrics(v GCMectrics) Option { + return func(c *cfg) { + c.gcCfg.metrics = v + } +} + // WithReportErrorFunc returns option to specify callback for handling storage-related errors // in the background workers. func WithReportErrorFunc(f func(selfID string, message string, err error)) Option { diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go new file mode 100644 index 000000000..c4d5ecb50 --- /dev/null +++ b/pkg/metrics/gc.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + gcSubsystem = "garbage_collector" + gcShardID = "shard_id" + gcSuccess = "success" + gcStatus = "status" + gcDeleted = "deleted" + gcFailed = "failed_to_delete" + gcObjectType = "object_type" +) + +type GCMetrics interface { + AddRunDuration(shardID string, d time.Duration, success bool) + AddDeletedCount(shardID string, deleted, failed uint64) + AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) + AddInhumedObjectCount(shardID string, count uint64, objectType string) +} + +type gcMetrics struct { + runDuration metric[*prometheus.CounterVec] + deletedCounter metric[*prometheus.CounterVec] + expCollectDuration metric[*prometheus.CounterVec] + inhumedCounter metric[*prometheus.CounterVec] +} + +func (m *gcMetrics) register() { + mustRegister(m.runDuration) + mustRegister(m.deletedCounter) + mustRegister(m.expCollectDuration) + mustRegister(m.inhumedCounter) +} + +func newGCMetrics() *gcMetrics { + return &gcMetrics{ + runDuration: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "delete_duration_seconds", + Help: "The total time of GC runs to delete objects from disk", + }, []string{gcShardID, gcSuccess}), + deletedCounter: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "deleted_objects_count", + Help: "Total count of objects GC deleted or failed to delete from disk", + }, []string{gcShardID, gcStatus}), + expCollectDuration: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marking_duration_seconds", + Help: "The total time of GC runs to mark expired objects as removed", + }, []string{gcShardID, gcSuccess, gcObjectType}), + inhumedCounter: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marked_for_removal_objects_count", + Help: "Total count of expired objects GC marked to remove", + }, []string{gcShardID, gcObjectType}), + } +} + +func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { + m.runDuration.value.With(prometheus.Labels{ + gcShardID: shardID, + gcSuccess: fmt.Sprintf("%v", success), + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { + m.deletedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcStatus: gcDeleted, + }).Add(float64(deleted)) + m.deletedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcStatus: gcFailed, + }).Add(float64(failed)) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { + m.expCollectDuration.value.With(prometheus.Labels{ + gcShardID: shardID, + gcSuccess: fmt.Sprintf("%v", success), + gcObjectType: objectType, + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { + m.inhumedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcObjectType: objectType, + }).Add(float64(count)) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index cca82b5fe..526e460c5 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -16,6 +16,7 @@ type NodeMetrics struct { writeCacheMetrics *writeCacheMetrics treeService *treeServiceMetrics epoch metric[prometheus.Gauge] + gc *gcMetrics } func NewNodeMetrics() *NodeMetrics { @@ -45,6 +46,9 @@ func NewNodeMetrics() *NodeMetrics { writeCacheMetrics := newWriteCacheMetrics() writeCacheMetrics.register() + gc := newGCMetrics() + gc.register() + return &NodeMetrics{ objectServiceMetrics: objectService, engineMetrics: engine, @@ -53,6 +57,7 @@ func NewNodeMetrics() *NodeMetrics { treeService: treeService, epoch: epoch, writeCacheMetrics: writeCacheMetrics, + gc: gc, } } @@ -72,3 +77,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { func (m *NodeMetrics) TreeService() tree.MetricsRegister { return m.treeService } + +func (m *NodeMetrics) GC() GCMetrics { + return m.gc +} -- 2.45.2 From dbf41391b55992decff4e55b9c3480874af3cde3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 13:24:30 +0300 Subject: [PATCH 077/233] [#401] engine: Extend evacuation logs Add operation-tag to logger. Log evacuation results. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index d698fca74..b0a7e77b4 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -21,7 +21,11 @@ import ( "go.uber.org/zap" ) -var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") +var ( + ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") + + evacuationOperationLogField = zap.String("operation", "evacuation") +) // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { @@ -196,22 +200,28 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.evacuateLimiter.Complete(err) }() - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField) return err } for _, shardID := range shardIDs { if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) return err } } - e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, + zap.Strings("shard_ids", shardIDs), + evacuationOperationLogField, + zap.Uint64("total", res.Total()), + zap.Uint64("evacuated", res.Evacuated()), + zap.Uint64("failed", res.Failed()), + ) return nil } @@ -256,7 +266,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) return err } @@ -332,7 +342,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to res.failed.Add(1) continue } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } @@ -353,7 +363,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } res.evacuated.Add(1) @@ -381,7 +391,8 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), - zap.Stringer("addr", addr)) + zap.Stringer("addr", addr), + evacuationOperationLogField) } return true, nil } -- 2.45.2 From 4476a1dbaf20bc244ea81e22865ae6898c83c6c7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 31 May 2023 15:37:18 +0300 Subject: [PATCH 078/233] [#413] morph/client: Fix govet warnings Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index de29d0c03..6a7a5b51a 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -90,7 +90,7 @@ type cache struct { txHeights *lru.Cache[util.Uint256, uint32] } -func (c cache) nns() *util.Uint160 { +func (c *cache) nns() *util.Uint160 { c.m.RLock() defer c.m.RUnlock() @@ -104,7 +104,7 @@ func (c *cache) setNNSHash(nnsHash util.Uint160) { c.nnsHash = &nnsHash } -func (c cache) groupKey() *keys.PublicKey { +func (c *cache) groupKey() *keys.PublicKey { c.m.RLock() defer c.m.RUnlock() -- 2.45.2 From f7c4c0745318837e5104ed9a06b3a5572d4223eb Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 1 Jun 2023 10:07:48 +0300 Subject: [PATCH 079/233] [#xx] Create innerring metrics before metrics usage Signed-off-by: Alejandro Lopez --- pkg/innerring/innerring.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 3a690bbbe..cbcb4699a 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -326,7 +326,10 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error - server := &Server{log: log} + server := &Server{ + log: log, + metrics: metrics.NewInnerRingMetrics(), + } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) @@ -393,8 +396,6 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.metrics = metrics.NewInnerRingMetrics() - return server, nil } -- 2.45.2 From 7e9a1f394a62b4e274ea5b93cd67ce3a89c43124 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:19:59 +0300 Subject: [PATCH 080/233] [#412] node: Update deps Signed-off-by: Dmitrii Stepanov --- go.mod | 24 ++++++++++++++---------- go.sum | Bin 97348 -> 98963 bytes 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 2c95bd41b..6c49c02b4 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 @@ -23,15 +24,14 @@ require ( github.com/panjf2000/ants/v2 v2.7.4 github.com/paulmach/orb v0.9.2 github.com/prometheus/client_golang v1.15.1 - github.com/prometheus/client_model v0.4.0 github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.3 go.etcd.io/bbolt v1.3.7 - go.opentelemetry.io/otel v1.15.1 - go.opentelemetry.io/otel/trace v1.15.1 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 @@ -58,6 +58,8 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -83,6 +85,7 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.43.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -95,11 +98,12 @@ require ( github.com/twmb/murmur3 v1.1.7 // indirect github.com/urfave/cli v1.22.13 // indirect go.mongodb.org/mongo-driver v1.11.6 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect - go.opentelemetry.io/otel/sdk v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index d728851b9b08097b80e81aa9be1e3b40cd9eaf61..b9b3fde4125d22a606b0cca07094d61b19dfcee4 100644 GIT binary patch delta 2264 zcmb7_%dg{B9mf?O1r>FuD3=8zs_6m=O_Z^ZkDt|ml*D=2&clzyj$4WGKN?F_;AjBTTU0jw(umq+>V#TWeK$jIrsKObPa_3SBRo-~>`+k1E@A;kc{o_Br zeEP{Nzj^u^4*>uqPSP!!K(0%(9?4kFqX4{Z(U|E_9klYIjbEt2-i{()aC<1&jyXRI z)>10jqCD)*@zal9>xiEzg!%E4P3)fa5>XHoY1$2R^2p0sI|qneW7$?0qm_^+>u}~( z3kq)i)u`6{-s$#1^XA!uUpIaj+Zjqr%MPP3KRhb&C-v7Na022K1UO4DB++s>#w7p= z0J7;9%@1T+w%S__r833uD4W1Mj-Si9x|^*{6SLRj3LA=r)Q!(D@AES5%%!uFEzV*Q zZOP0$8reyo>1T3WFxnz<6dsrw5C^1|_JiJwxxW7TH+hHyNCARq0i0wB&$E#W?Q`G9 zDjoA3u`LZyZ-lEwHK~+npkqQ}+8Ckehy9#kB`T7_>>_scnPNA;d;Oo^y&?dELs*}0 zi>6r{5FQ0N?xsus{jn9d{4{Y|o6vP5w6{`(C23rSE=sfW-v;FYDzZl&$9rwD1^$P#ZyuCr6a`cClqDR-P*yM{eIs5?hS zS{3g6NM|f7&^EexfHOa&)ujaKLn)#|`wW}@FRUaUmBeKLcpf0;D3K2rm<=j-U2K-r z(v}n)+Y-%vrD5 zj1Cf9t!7mOiQigptqJ-+j+)G1(KfL8x^2wF?7)$uc{8N*Ad^*cYcozm-?2qN>XIU5iRg&m};0^k+sC*)8GE| z#)Y1f=3bf2n`o0d<(NPmKXmba=HY@iTR667VVywZ`#5y*ODT5`E?KGV)2urrbcUBSt-Gd6gp$@D7Fislp0>KM9hpDt!GJsnHv}qe%cDx9>BOy}Groq$C{_^R> z+Cok++m01-6PT)1Hpi}SblAS71Tj#$?0mo2LJcoh3+GaL4?cph%tr{%6pr3|>D&pU z(NT)qa3c)^Wpm_IPD8i~eLghLgllRVL_n^VfKXzuCM;@k=ao=~%Qo?MJBor(Rx7uML=b;`38C1=4+Yi5A-`_NJ zF3WP>e8o-O;U?ESA^JTSp@4VGEBT1t3V6_R02 zIx5`re840WL-^@&l6J=7SoeFE(tAGS<7`xjMp2CPS*JZshfOrh%RrNw>NfA^MxT>; znzd$h@&f)ePb~-CwSg6s#G2auSjUs9Sj?`FqBtKThQ5+kaf*oL5=0}>OAbe zByh)tWzLn$9Z=oL5mxTXG(~3b7$w-1M6#FGyOfX@dt~9rJKvmdo!K`Og~ zH@^N*XmSQC_7|(htJl8|8(+I! JgpH5B@o!sNegEszUN9vExJHQykA~?D&wxaqQUs$B9>L z*bmADu;2nf%py0y7Kwt?OF?B*F*8h1X4DIKpYQqK|NFMzKlttK(J!wa@(7I}IE-{& zz!S3LQ&AXsIC4BiUaqTNIk3l+(ZJK%k-8p&Q?{`w9l+yDLoD;5=DUnNL8^HB-9uJ> zFQ^@7CutU3^T=Ar)yS-QVy#uW>yAXnvBWxe!TAQHn`Lz>G?KzfV>`RzeY4#?f-cV< z{oL9@6TX$Vi-{g>DF8B|80? z*~ra!JeF>#SFlgt3%mUN@jqW(N>uuM`!4w8@~7bK93@U}n!u!bg)Jh3-3-0Dgd@_o zv8BhX1_rVuq>y(48l&OF5j|>q@#eGb@#_!2Jw8WHkgNPb0jVhV1aZVG-hy-sSxpmK zS#h|Bm0co+A~7neN|#tfvnIEgr`y3FPxcmLm)9$;34GMTfr{h_K+_HxnkwIqSJOU3 z%1yG0b(T@?#Ob>dU|@|p1f&d_5j5aah4Q9`;v`)L3-yV~p&nEEYYE^1NNy#**b7vf zEcEOtWeBpj>7e|`Elq)SM+gN)y@3`&t~>Mw>sZNzee~XN(KeUGT-Om=9@7Fh&-FEk z$}WV>G9GIrbEqynU=2?J8s8E=-(LLre*?J`Wj#V-PPskC>iY0b%-yMyONH2O<|>6-Ta$i8qG;&m`mainR%rDfiKP!&ek!O+AQEtm!;7ZHjeh z4A>k|6K#lJx}qwxYHn~Pn8CMCIC@zI$$p!zH=gLq22>_--keW|x+n?Jq*mQI#}){# zI8BKGb0yF8MzEwkf;m7OEF@Kos!e(H;Iv}$rau?V3#RVTj6)>fl2aqJ6>3d_u% z^r|YnNqDFO#!5!c#JqC3?>zluy@4Qy)hX=c}k6N0$ W`_gZ{`*>HUtsh&vi>&p*lYaqb_M+PW -- 2.45.2 From 74578052f9ac750c3f66b2ff4197d50096464f4c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:24:04 +0300 Subject: [PATCH 081/233] [#412] node: Replace tracing package Use observability module. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/config/tracing/config.go | 2 +- cmd/frostfs-node/tracing.go | 2 +- pkg/local_object_storage/blobovnicza/delete.go | 2 +- pkg/local_object_storage/blobovnicza/get.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/delete.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/exists.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/get.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/get_range.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/put.go | 2 +- pkg/local_object_storage/blobstor/delete.go | 2 +- pkg/local_object_storage/blobstor/exists.go | 2 +- pkg/local_object_storage/blobstor/fstree/fstree.go | 2 +- pkg/local_object_storage/blobstor/get.go | 2 +- pkg/local_object_storage/blobstor/get_range.go | 2 +- pkg/local_object_storage/blobstor/put.go | 2 +- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 2 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/lock.go | 2 +- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/engine/select.go | 2 +- pkg/local_object_storage/engine/tree.go | 2 +- pkg/local_object_storage/engine/writecache.go | 2 +- pkg/local_object_storage/metabase/delete.go | 2 +- pkg/local_object_storage/metabase/exists.go | 2 +- pkg/local_object_storage/metabase/get.go | 2 +- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/lock.go | 2 +- pkg/local_object_storage/metabase/movable.go | 2 +- pkg/local_object_storage/metabase/put.go | 2 +- pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/metabase/storage_id.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 2 +- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/count.go | 2 +- pkg/local_object_storage/shard/delete.go | 2 +- pkg/local_object_storage/shard/exists.go | 2 +- pkg/local_object_storage/shard/get.go | 2 +- pkg/local_object_storage/shard/head.go | 2 +- pkg/local_object_storage/shard/inhume.go | 2 +- pkg/local_object_storage/shard/lock.go | 2 +- pkg/local_object_storage/shard/move.go | 2 +- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/range.go | 2 +- pkg/local_object_storage/shard/select.go | 2 +- pkg/local_object_storage/shard/tree.go | 2 +- pkg/local_object_storage/shard/writecache.go | 2 +- pkg/local_object_storage/writecache/delete.go | 2 +- pkg/local_object_storage/writecache/flush.go | 2 +- pkg/local_object_storage/writecache/get.go | 2 +- pkg/local_object_storage/writecache/mode.go | 2 +- pkg/local_object_storage/writecache/put.go | 2 +- pkg/services/object/get/local.go | 2 +- pkg/services/object/get/remote.go | 2 +- pkg/services/object/get/v2/get_forwarder.go | 2 +- pkg/services/object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/internal/client/client.go | 2 +- pkg/services/object/put/v2/streamer.go | 2 +- pkg/services/tree/redirect.go | 2 +- pkg/services/tree/replicator.go | 2 +- 65 files changed, 65 insertions(+), 65 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 00ade7ce2..f8605c21e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -16,7 +16,6 @@ import ( "time" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" @@ -60,6 +59,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go index 76572cc31..e846be158 100644 --- a/cmd/frostfs-node/config/tracing/config.go +++ b/cmd/frostfs-node/config/tracing/config.go @@ -1,9 +1,9 @@ package tracing import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" ) const ( diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index d963ba866..08dc049da 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.uber.org/zap" ) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 29a587cc9..e880815a6 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -3,8 +3,8 @@ package blobovnicza import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index c1cd19e53..ff29358b0 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 5aa9062a6..1e38c6ea1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -5,11 +5,11 @@ import ( "encoding/hex" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index e7852612b..3324507cf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -5,10 +5,10 @@ import ( "encoding/hex" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 8955eb148..3c8e288d4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -6,11 +6,11 @@ import ( "fmt" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index fb23a9671..6e9620adf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -7,11 +7,11 @@ import ( "path/filepath" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 4e1d6621f..95ed15540 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -5,10 +5,10 @@ import ( "errors" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 377214fb8..f1b14481c 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -5,8 +5,8 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 3c76764a9..760e7b2a4 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -4,9 +4,9 @@ import ( "context" "encoding/hex" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index b0879f68e..76487813e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -12,11 +12,11 @@ import ( "strings" "syscall" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 65bc87c07..eadb990c8 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -5,9 +5,9 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index ff9e72e97..ca4e41f33 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -6,9 +6,9 @@ import ( "errors" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 2ae7f0fe6..125b445b6 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index f9b9c9a87..2125fad72 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index b0a7e77b4..c103ede73 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -6,13 +6,13 @@ import ( "fmt" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 683b7bde8..d376198ba 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 130e76c3d..5da97bab5 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index b1204ed99..57ac52751 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 4562c1a57..61a5a0dc9 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 0543f9f15..4ac7f90f9 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,13 +4,13 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 3d119ac6f..29ec8b2bc 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -5,10 +5,10 @@ import ( "errors" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index e1039ea23..48d2be674 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -3,8 +3,8 @@ package engine import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index e7d66094c..6b8f83f31 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 692fa4be5..cd8278272 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -4,11 +4,11 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 5340f5d08..6ddf96593 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index cfd37b0d2..74faf1634 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -5,8 +5,8 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index e76b9d4a7..2f94c72d0 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index a6887a33b..e4af1d1b9 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 5c3c9720d..2b5fcd775 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -5,8 +5,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index 412c46393..033b9c718 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index bc6520a05..86830f7da 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -7,10 +7,10 @@ import ( "fmt" gio "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 2ced6c4b7..b6b8f5b10 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -8,8 +8,8 @@ import ( "strings" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 794879a3f..b53010a1e 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1ecc89cb5..9b62f0649 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -12,10 +12,10 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e8e2bd4d7..84efa1d31 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go index b68c2f43e..abed5278e 100644 --- a/pkg/local_object_storage/shard/count.go +++ b/pkg/local_object_storage/shard/count.go @@ -3,7 +3,7 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 4843314de..2c7e0af27 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -4,11 +4,11 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 66c61fccc..60809da6a 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 5268ac790..589ec53c9 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index a15cdfdca..a0ec231af 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -3,8 +3,8 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 12a2900ac..6a2f9311d 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index cfbd94c5b..52186cbfd 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 119910623..9832c9c84 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index d7d4ae538..79dc4846e 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 06aea2f8a..895dc2fe0 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -4,11 +4,11 @@ import ( "context" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 7f776c18a..2d4d473b4 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index d5b3b67bf..ad89fa633 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 245eb4c70..7ce279c54 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index f5a292ed4..796fda623 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 767435ebf..da8b3a0fa 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -6,12 +6,12 @@ import ( "errors" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index f8f6de9b0..aac5759ae 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 7e9373a42..bdbbec7c9 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -5,10 +5,10 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 1e99e4a28..61e81ec30 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -5,9 +5,9 @@ import ( "errors" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 62dde3281..03ede58cc 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -4,8 +4,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 69bdbf271..e3464f941 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index bc27f69a2..580c0b58c 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 1137ba33c..7858e2b53 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 1b0374aea..fa1506435 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -6,7 +6,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -15,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 6beb67476..8214c784d 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -8,8 +8,8 @@ import ( "fmt" "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 65531dc66..9c6de4ca8 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -16,6 +15,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 3de71b554..0afd3439a 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -5,8 +5,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 7199dc40e..0ca30273e 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -8,9 +8,9 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" -- 2.45.2 From c09144ecf155758ec65d92fa8d4506ce0db68ee4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:25:32 +0300 Subject: [PATCH 082/233] [#412] node: Replace metrics package Use observability module. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/metrics.go | 2 +- cmd/frostfs-node/metrics.go | 2 +- pkg/metrics/desc.go | 109 --------------------------------- pkg/metrics/desc_test.go | 65 -------------------- pkg/metrics/engine.go | 79 ++++++++++-------------- pkg/metrics/gc.go | 34 +++++----- pkg/metrics/innerring.go | 23 +++---- pkg/metrics/node.go | 15 ++--- pkg/metrics/object.go | 100 +++++++++++------------------- pkg/metrics/registry.go | 42 ------------- pkg/metrics/replicator.go | 33 +++++----- pkg/metrics/state.go | 15 +++-- pkg/metrics/treeservice.go | 25 +++----- pkg/metrics/writecache.go | 64 ++++++++----------- scripts/export-metrics/main.go | 13 ++-- 15 files changed, 162 insertions(+), 459 deletions(-) delete mode 100644 pkg/metrics/desc.go delete mode 100644 pkg/metrics/desc_test.go delete mode 100644 pkg/metrics/registry.go diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go index 39b432c74..dd982b780 100644 --- a/cmd/frostfs-ir/metrics.go +++ b/cmd/frostfs-ir/metrics.go @@ -1,7 +1,7 @@ package main import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) func newMetricsComponent() *httpComponent { diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index cf621086d..19b4af51f 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -2,7 +2,7 @@ package main import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) func metricsComponent(c *cfg) (*httpComponent, bool) { diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go deleted file mode 100644 index 612435b2f..000000000 --- a/pkg/metrics/desc.go +++ /dev/null @@ -1,109 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -type metric[T prometheus.Collector] struct { - value T - desc Description -} - -// Descriptions contains metric description suitable for further processing. -// The only reason for it to exist is `prometheus.Desc` disallowing field access directly. -// https://github.com/prometheus/client_golang/pull/326 -// https://github.com/prometheus/client_golang/issues/516 -// https://github.com/prometheus/client_golang/issues/222 -type Description struct { - Name string `json:"name"` - Help string `json:"help"` - Type string `json:"type"` - ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` - VariableLabels []string `json:"variable_labels,omitempty"` -} - -func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] { - return metric[prometheus.Gauge]{ - value: prometheus.NewGauge(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] { - return metric[*prometheus.GaugeVec]{ - value: prometheus.NewGaugeVec(opts, labelNames), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labelNames, - }, - } -} - -func newGaugeFunc(opts prometheus.GaugeOpts, f func() float64) metric[prometheus.GaugeFunc] { - return metric[prometheus.GaugeFunc]{ - value: prometheus.NewGaugeFunc(opts, f), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { - return metric[prometheus.Counter]{ - value: prometheus.NewCounter(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_COUNTER.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newCounterVec(opts prometheus.CounterOpts, labels []string) metric[*prometheus.CounterVec] { - return metric[*prometheus.CounterVec]{ - value: prometheus.NewCounterVec(opts, labels), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_COUNTER.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labels, - }, - } -} - -func newHistogramVec(opts prometheus.HistogramOpts, labelNames []string) metric[*prometheus.HistogramVec] { - return metric[*prometheus.HistogramVec]{ - value: prometheus.NewHistogramVec(opts, labelNames), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_HISTOGRAM.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labelNames, - }, - } -} - -// DescribeAll returns descriptions for all registered metrics. -func DescribeAll() ([]Description, error) { - registeredDescriptionsMtx.Lock() - defer registeredDescriptionsMtx.Unlock() - - ds := make([]Description, len(registeredDescriptions)) - copy(ds, registeredDescriptions) - return ds, nil -} diff --git a/pkg/metrics/desc_test.go b/pkg/metrics/desc_test.go deleted file mode 100644 index 28b5e2132..000000000 --- a/pkg/metrics/desc_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package metrics - -import ( - "strings" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" -) - -func TestDescribeAll(t *testing.T) { - const ( - namespace = "my_ns" - subsystem = "mysub" - ) - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_counter", - })) - - labels := []string{"label1", "label2"} - mustRegister(newGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_gauge", - }, labels)) - - constLabels := prometheus.Labels{ - "const1": "abc", - "const2": "xyz", - } - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "with_const_labels", - ConstLabels: constLabels, - })) - - descriptions, err := DescribeAll() - require.NoError(t, err) - - seen := make(map[string]bool) - for i := range descriptions { - if !strings.HasPrefix(descriptions[i].Name, namespace) { - continue - } - - require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name) - seen[descriptions[i].Name] = true - - switch descriptions[i].Name { - case prometheus.BuildFQName(namespace, subsystem, "my_counter"): - require.True(t, len(descriptions[i].VariableLabels) == 0) - case prometheus.BuildFQName(namespace, subsystem, "my_gauge"): - require.Equal(t, labels, descriptions[i].VariableLabels) - case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"): - require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels)) - require.Equal(t, constLabels, descriptions[i].ConstantLabels) - default: - require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name) - } - } - require.Equal(t, 3, len(seen), "not all registered metrics were iterated over") -} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 28fc1e028..4e78f4ac2 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -5,24 +5,25 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type ( engineMetrics struct { - listContainersDuration metric[prometheus.Counter] - estimateContainerSizeDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - existsDuration metric[prometheus.Counter] - getDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - inhumeDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - listObjectsDuration metric[prometheus.Counter] - containerSize metric[*prometheus.GaugeVec] - payloadSize metric[*prometheus.GaugeVec] + listContainersDuration prometheus.Counter + estimateContainerSizeDuration prometheus.Counter + deleteDuration prometheus.Counter + existsDuration prometheus.Counter + getDuration prometheus.Counter + headDuration prometheus.Counter + inhumeDuration prometheus.Counter + putDuration prometheus.Counter + rangeDuration prometheus.Counter + searchDuration prometheus.Counter + listObjectsDuration prometheus.Counter + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec } ) @@ -46,8 +47,8 @@ func newEngineMetrics() engineMetrics { } } -func newEngineCounter(name, help string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newEngineCounter(name, help string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -55,15 +56,15 @@ func newEngineCounter(name, help string) metric[prometheus.Counter] { }) } -func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] { +func newEngineMethodDurationCounter(method string) prometheus.Counter { return newEngineCounter( fmt.Sprintf("%s_duration", method), fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), ) } -func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -71,70 +72,54 @@ func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus }, labels) } -func (m engineMetrics) register() { - mustRegister(m.listContainersDuration) - mustRegister(m.estimateContainerSizeDuration) - mustRegister(m.deleteDuration) - mustRegister(m.existsDuration) - mustRegister(m.getDuration) - mustRegister(m.headDuration) - mustRegister(m.inhumeDuration) - mustRegister(m.putDuration) - mustRegister(m.rangeDuration) - mustRegister(m.searchDuration) - mustRegister(m.listObjectsDuration) - mustRegister(m.containerSize) - mustRegister(m.payloadSize) -} - func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) + m.listObjectsDuration.Add(float64(d)) } func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.value.Add(float64(d)) + m.estimateContainerSizeDuration.Add(float64(d)) } func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) + m.deleteDuration.Add(float64(d)) } func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.value.Add(float64(d)) + m.existsDuration.Add(float64(d)) } func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) + m.getDuration.Add(float64(d)) } func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) + m.headDuration.Add(float64(d)) } func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.value.Add(float64(d)) + m.inhumeDuration.Add(float64(d)) } func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) + m.putDuration.Add(float64(d)) } func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) + m.rangeDuration.Add(float64(d)) } func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) + m.searchDuration.Add(float64(d)) } func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) + m.listObjectsDuration.Add(float64(d)) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { - m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) + m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index c4d5ecb50..2457c0c6b 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -25,40 +26,33 @@ type GCMetrics interface { } type gcMetrics struct { - runDuration metric[*prometheus.CounterVec] - deletedCounter metric[*prometheus.CounterVec] - expCollectDuration metric[*prometheus.CounterVec] - inhumedCounter metric[*prometheus.CounterVec] -} - -func (m *gcMetrics) register() { - mustRegister(m.runDuration) - mustRegister(m.deletedCounter) - mustRegister(m.expCollectDuration) - mustRegister(m.inhumedCounter) + runDuration *prometheus.CounterVec + deletedCounter *prometheus.CounterVec + expCollectDuration *prometheus.CounterVec + inhumedCounter *prometheus.CounterVec } func newGCMetrics() *gcMetrics { return &gcMetrics{ - runDuration: newCounterVec(prometheus.CounterOpts{ + runDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "delete_duration_seconds", Help: "The total time of GC runs to delete objects from disk", }, []string{gcShardID, gcSuccess}), - deletedCounter: newCounterVec(prometheus.CounterOpts{ + deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "deleted_objects_count", Help: "Total count of objects GC deleted or failed to delete from disk", }, []string{gcShardID, gcStatus}), - expCollectDuration: newCounterVec(prometheus.CounterOpts{ + expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marking_duration_seconds", Help: "The total time of GC runs to mark expired objects as removed", }, []string{gcShardID, gcSuccess, gcObjectType}), - inhumedCounter: newCounterVec(prometheus.CounterOpts{ + inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marked_for_removal_objects_count", @@ -68,19 +62,19 @@ func newGCMetrics() *gcMetrics { } func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { - m.runDuration.value.With(prometheus.Labels{ + m.runDuration.With(prometheus.Labels{ gcShardID: shardID, gcSuccess: fmt.Sprintf("%v", success), }).Add(d.Seconds()) } func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { - m.deletedCounter.value.With( + m.deletedCounter.With( prometheus.Labels{ gcShardID: shardID, gcStatus: gcDeleted, }).Add(float64(deleted)) - m.deletedCounter.value.With( + m.deletedCounter.With( prometheus.Labels{ gcShardID: shardID, gcStatus: gcFailed, @@ -88,7 +82,7 @@ func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { } func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { - m.expCollectDuration.value.With(prometheus.Labels{ + m.expCollectDuration.With(prometheus.Labels{ gcShardID: shardID, gcSuccess: fmt.Sprintf("%v", success), gcObjectType: objectType, @@ -96,7 +90,7 @@ func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Du } func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { - m.inhumedCounter.value.With( + m.inhumedCounter.With( prometheus.Labels{ gcShardID: shardID, gcObjectType: objectType, diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index bff9184ec..9d8b76bf9 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -4,6 +4,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -15,27 +16,27 @@ const ( // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch metric[prometheus.Gauge] - health metric[prometheus.Gauge] - eventDuration metric[*prometheus.HistogramVec] + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( - epoch = newGauge(prometheus.GaugeOpts{ + epoch = metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - health = newGauge(prometheus.GaugeOpts{ + health = metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "health", Help: "Current inner-ring node state.", }) - eventDuration = newHistogramVec(prometheus.HistogramOpts{ + eventDuration = metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "event_duration_seconds", @@ -43,10 +44,6 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { }, []string{innerRingLabelType, innerRingLabelSuccess}) ) - mustRegister(epoch) - mustRegister(health) - mustRegister(eventDuration) - return &InnerRingServiceMetrics{ epoch: epoch, health: health, @@ -56,16 +53,16 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { // SetEpoch updates epoch metrics. func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) + m.epoch.Set(float64(epoch)) } // SetHealth updates health metrics. func (m InnerRingServiceMetrics) SetHealth(s int32) { - m.health.value.Set(float64(s)) + m.health.Set(float64(s)) } func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { - m.eventDuration.value.With(prometheus.Labels{ + m.eventDuration.With(prometheus.Labels{ innerRingLabelType: typ, innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 526e460c5..8819ba15b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -2,6 +2,7 @@ package metrics import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -15,39 +16,31 @@ type NodeMetrics struct { writeCacheMetrics *writeCacheMetrics treeService *treeServiceMetrics - epoch metric[prometheus.Gauge] + epoch prometheus.Gauge gc *gcMetrics } func NewNodeMetrics() *NodeMetrics { objectService := newObjectServiceMetrics() - objectService.register() engine := newEngineMetrics() - engine.register() state := newStateMetrics() - state.register() replicator := newReplicatorMetrics() - replicator.register() treeService := newTreeServiceMetrics() - treeService.register() - epoch := newGauge(prometheus.GaugeOpts{ + epoch := metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - mustRegister(epoch) writeCacheMetrics := newWriteCacheMetrics() - writeCacheMetrics.register() gc := newGCMetrics() - gc.register() return &NodeMetrics{ objectServiceMetrics: objectService, @@ -63,7 +56,7 @@ func NewNodeMetrics() *NodeMetrics { // SetEpoch updates epoch metric. func (m *NodeMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) + m.epoch.Set(float64(epoch)) } // WriteCache returns WriteCache metrics. diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 5ec575749..87916414d 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -12,8 +13,8 @@ const objectSubsystem = "object" type ( methodCount struct { - success metric[prometheus.Counter] - total metric[prometheus.Counter] + success prometheus.Counter + total prometheus.Counter } objectServiceMetrics struct { @@ -25,19 +26,19 @@ type ( rangeCounter methodCount rangeHashCounter methodCount - getDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - rangeHashDuration metric[prometheus.Counter] + getDuration prometheus.Counter + putDuration prometheus.Counter + headDuration prometheus.Counter + searchDuration prometheus.Counter + deleteDuration prometheus.Counter + rangeDuration prometheus.Counter + rangeHashDuration prometheus.Counter - putPayload metric[prometheus.Counter] - getPayload metric[prometheus.Counter] + putPayload prometheus.Counter + getPayload prometheus.Counter - shardMetrics metric[*prometheus.GaugeVec] - shardsReadonly metric[*prometheus.GaugeVec] + shardMetrics *prometheus.GaugeVec + shardsReadonly *prometheus.GaugeVec } ) @@ -49,13 +50,13 @@ const ( func newObjectMethodCallCounter(name string) methodCount { return methodCount{ - success: newCounter(prometheus.CounterOpts{ + success: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count_success", name), Help: fmt.Sprintf("The number of successful %s requests processed", name), }), - total: newCounter(prometheus.CounterOpts{ + total: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count", name), @@ -64,15 +65,10 @@ func newObjectMethodCallCounter(name string) methodCount { } } -func (m methodCount) mustRegister() { - mustRegister(m.success) - mustRegister(m.total) -} - func (m methodCount) Inc(success bool) { - m.total.value.Inc() + m.total.Inc() if success { - m.success.value.Inc() + m.success.Inc() } } @@ -99,8 +95,8 @@ func newObjectServiceMetrics() objectServiceMetrics { } } -func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newObjectMethodPayloadCounter(method string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_payload", method), @@ -108,8 +104,8 @@ func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { }) } -func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newObjectMethodDurationCounter(method string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_duration", method), @@ -117,8 +113,8 @@ func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { }) } -func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: name, @@ -126,30 +122,6 @@ func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus }, labels) } -func (m objectServiceMetrics) register() { - m.getCounter.mustRegister() - m.putCounter.mustRegister() - m.headCounter.mustRegister() - m.searchCounter.mustRegister() - m.deleteCounter.mustRegister() - m.rangeCounter.mustRegister() - m.rangeHashCounter.mustRegister() - - mustRegister(m.getDuration) - mustRegister(m.putDuration) - mustRegister(m.headDuration) - mustRegister(m.searchDuration) - mustRegister(m.deleteDuration) - mustRegister(m.rangeDuration) - mustRegister(m.rangeHashDuration) - - mustRegister(m.putPayload) - mustRegister(m.getPayload) - - mustRegister(m.shardMetrics) - mustRegister(m.shardsReadonly) -} - func (m objectServiceMetrics) IncGetReqCounter(success bool) { m.getCounter.Inc(success) } @@ -179,43 +151,43 @@ func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { } func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) + m.getDuration.Add(float64(d)) } func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) + m.putDuration.Add(float64(d)) } func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) + m.headDuration.Add(float64(d)) } func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) + m.searchDuration.Add(float64(d)) } func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) + m.deleteDuration.Add(float64(d)) } func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) + m.rangeDuration.Add(float64(d)) } func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.value.Add(float64(d)) + m.rangeHashDuration.Add(float64(d)) } func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.value.Add(float64(ln)) + m.putPayload.Add(float64(ln)) } func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.value.Add(float64(ln)) + m.getPayload.Add(float64(ln)) } func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.value.With( + m.shardMetrics.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -224,7 +196,7 @@ func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, del } func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.value.With( + m.shardMetrics.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -237,7 +209,7 @@ func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { if readonly { flag = 1 } - m.shardsReadonly.value.With( + m.shardsReadonly.With( prometheus.Labels{ shardIDLabelKey: shardID, }, diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go deleted file mode 100644 index eef613d04..000000000 --- a/pkg/metrics/registry.go +++ /dev/null @@ -1,42 +0,0 @@ -package metrics - -import ( - "net/http" - "sync" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -// Handler returns an http.Handler for the local registry. -func Handler() http.Handler { - promhttp.Handler() - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) -} - -var ( - registry = prometheus.NewRegistry() - // registeredDescriptionsMtx protects collectors slice. - // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. - registeredDescriptionsMtx sync.Mutex - registeredDescriptions []Description -) - -func init() { - registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - registry.MustRegister(collectors.NewGoCollector()) -} - -func mustRegister[T prometheus.Collector](cs ...metric[T]) { - for i := range cs { - registry.MustRegister(cs[i].value) - } - registeredDescriptionsMtx.Lock() - for i := range cs { - registeredDescriptions = append(registeredDescriptions, cs[i].desc) - } - registeredDescriptionsMtx.Unlock() -} diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index 55f736c66..0deafe916 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -1,29 +1,32 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) const replicatorSubsystem = "replicator" type replicatorMetrics struct { - inFlightRequests metric[prometheus.Gauge] - processedObjects metric[prometheus.Counter] - totalReplicatedPayloadSize metric[prometheus.Counter] + inFlightRequests prometheus.Gauge + processedObjects prometheus.Counter + totalReplicatedPayloadSize prometheus.Counter } func (m replicatorMetrics) IncInFlightRequest() { - m.inFlightRequests.value.Inc() + m.inFlightRequests.Inc() } func (m replicatorMetrics) DecInFlightRequest() { - m.inFlightRequests.value.Dec() + m.inFlightRequests.Dec() } func (m replicatorMetrics) IncProcessedObjects() { - m.processedObjects.value.Inc() + m.processedObjects.Inc() } func (m replicatorMetrics) AddPayloadSize(size int64) { - m.totalReplicatedPayloadSize.value.Add(float64(size)) + m.totalReplicatedPayloadSize.Add(float64(size)) } func newReplicatorMetrics() replicatorMetrics { @@ -34,14 +37,8 @@ func newReplicatorMetrics() replicatorMetrics { } } -func (m replicatorMetrics) register() { - mustRegister(m.inFlightRequests) - mustRegister(m.processedObjects) - mustRegister(m.totalReplicatedPayloadSize) -} - -func newReplicatorCounter(name, help string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newReplicatorCounter(name, help string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: replicatorSubsystem, Name: name, @@ -49,8 +46,8 @@ func newReplicatorCounter(name, help string) metric[prometheus.Counter] { }) } -func newReplicatorGauge(name, help string) metric[prometheus.Gauge] { - return newGauge(prometheus.GaugeOpts{ +func newReplicatorGauge(name, help string) prometheus.Gauge { + return metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: replicatorSubsystem, Name: name, diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index dce0402cd..893849911 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -1,16 +1,19 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) const stateSubsystem = "state" type stateMetrics struct { - healthCheck metric[prometheus.Gauge] + healthCheck prometheus.Gauge } func newStateMetrics() stateMetrics { return stateMetrics{ - healthCheck: newGauge(prometheus.GaugeOpts{ + healthCheck: metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, Name: "health", @@ -19,10 +22,6 @@ func newStateMetrics() stateMetrics { } } -func (m stateMetrics) register() { - mustRegister(m.healthCheck) -} - func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.value.Set(float64(s)) + m.healthCheck.Set(float64(s)) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 135f6e6d2..903ef3496 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -4,33 +4,34 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) const treeServiceLabelSuccess = "success" type treeServiceMetrics struct { - replicateTaskDuration metric[*prometheus.HistogramVec] - replicateWaitDuration metric[*prometheus.HistogramVec] - syncOpDuration metric[*prometheus.HistogramVec] + replicateTaskDuration *prometheus.HistogramVec + replicateWaitDuration *prometheus.HistogramVec + syncOpDuration *prometheus.HistogramVec } func newTreeServiceMetrics() *treeServiceMetrics { const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ - replicateTaskDuration: newHistogramVec(prometheus.HistogramOpts{ + replicateTaskDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_task_duration_seconds", Help: "Duration of individual replication tasks executed as part of replication loops", }, []string{treeServiceLabelSuccess}), - replicateWaitDuration: newHistogramVec(prometheus.HistogramOpts{ + replicateWaitDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_wait_duration_seconds", Help: "Duration of overall waiting time for replication loops", }, []string{treeServiceLabelSuccess}), - syncOpDuration: newHistogramVec(prometheus.HistogramOpts{ + syncOpDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "sync_duration_seconds", @@ -39,26 +40,20 @@ func newTreeServiceMetrics() *treeServiceMetrics { } } -func (m *treeServiceMetrics) register() { - mustRegister(m.replicateTaskDuration) - mustRegister(m.replicateWaitDuration) - mustRegister(m.syncOpDuration) -} - func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { - m.replicateTaskDuration.value.With(prometheus.Labels{ + m.replicateTaskDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { - m.replicateWaitDuration.value.With(prometheus.Labels{ + m.replicateWaitDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { - m.syncOpDuration.value.With(prometheus.Labels{ + m.syncOpDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 74c330842..3c56aa2ba 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -37,18 +38,18 @@ type WriteCacheMetrics interface { } type writeCacheMetrics struct { - getDuration metric[*prometheus.HistogramVec] - putDuration metric[*prometheus.HistogramVec] - deleteDuration metric[*prometheus.HistogramVec] + getDuration *prometheus.HistogramVec + putDuration *prometheus.HistogramVec + deleteDuration *prometheus.HistogramVec - flushCounter metric[*prometheus.CounterVec] - evictCounter metric[*prometheus.CounterVec] + flushCounter *prometheus.CounterVec + evictCounter *prometheus.CounterVec - actualCount metric[*prometheus.GaugeVec] + actualCount *prometheus.GaugeVec - estimatedSize metric[*prometheus.GaugeVec] + estimatedSize *prometheus.GaugeVec - modeMetrics map[shardIDMode]metric[prometheus.GaugeFunc] + modeMetrics map[shardIDMode]prometheus.GaugeFunc modeValues map[string]string modeMtx sync.RWMutex } @@ -63,46 +64,46 @@ func newWriteCacheMetrics() *writeCacheMetrics { actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]metric[prometheus.GaugeFunc]), + modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), modeValues: make(map[string]string), } } func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.getDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.getDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.deleteDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.deleteDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.putDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.putDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Inc() } func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Dec() } func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Set(float64(count)) } func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { - m.estimatedSize.value.With(prometheus.Labels{ + m.estimatedSize.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Set(float64(size)) @@ -121,7 +122,7 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { return } - metric := newGaugeFunc( + metric := metrics.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: namespace, Subsystem: wcSubsystem, @@ -141,12 +142,11 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { } return 0 }) - mustRegister(metric) m.modeMetrics[key] = metric } func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { - m.flushCounter.value.With(prometheus.Labels{ + m.flushCounter.With(prometheus.Labels{ wcShardID: shardID, wcSuccess: fmt.Sprintf("%v", success), wcStorage: storageType, @@ -154,22 +154,12 @@ func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storag } func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { - m.evictCounter.value.With(prometheus.Labels{ + m.evictCounter.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Inc() } -func (m *writeCacheMetrics) register() { - mustRegister(m.getDuration) - mustRegister(m.putDuration) - mustRegister(m.deleteDuration) - mustRegister(m.actualCount) - mustRegister(m.estimatedSize) - mustRegister(m.flushCounter) - mustRegister(m.evictCounter) -} - func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { m.With( prometheus.Labels{ @@ -180,17 +170,17 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b ).Observe(float64(d)) } -func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { - return newHistogramVec(prometheus.HistogramOpts{ +func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { + return metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess, wcStorage}) + }, []string{wcShardID, wcSuccess}) } -func newWCOperationCounterVec(operation string, labels []string) metric[*prometheus.CounterVec] { - return newCounterVec(prometheus.CounterOpts{ +func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { + return metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_operation_count", operation), @@ -198,8 +188,8 @@ func newWCOperationCounterVec(operation string, labels []string) metric[*prometh }, labels) } -func newWCGaugeVec(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: name, diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 694eea38b..f29eca37c 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -6,7 +6,8 @@ import ( "fmt" "os" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) var ( @@ -25,10 +26,10 @@ func main() { var filename string switch { case *node != "": - _ = metrics.NewNodeMetrics() + _ = local_metrics.NewNodeMetrics() filename = *node case *ir != "": - _ = metrics.NewInnerRingMetrics() + _ = local_metrics.NewInnerRingMetrics() filename = *ir default: @@ -36,11 +37,7 @@ func main() { os.Exit(1) } - ds, err := metrics.DescribeAll() - if err != nil { - fmt.Fprintf(os.Stderr, "Could not parse metric descriptions: %v\n", err) - os.Exit(1) - } + ds := metrics.DescribeAll() data, err := json.Marshal(ds) if err != nil { -- 2.45.2 From dcdfb6ed41ba5f8edb104d563c0c0828cd8173c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:26:54 +0300 Subject: [PATCH 083/233] [#412] node: Use observability interceptors Use metrics and tracing interceptors. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/tree/client.go | 9 ++++++--- cmd/frostfs-node/grpc.go | 9 ++++++--- pkg/services/tree/cache.go | 9 ++++++--- pkg/services/tree/sync.go | 10 +++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index f25bff166..4f4f54657 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -5,10 +5,11 @@ import ( "strings" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -26,10 +27,12 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index b0a587782..b62ae9c45 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -7,10 +7,11 @@ import ( "net" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -22,10 +23,12 @@ func initGRPC(c *cfg) { serverOpts := []grpc.ServerOption{ grpc.MaxSendMsgSize(maxMsgSize), grpc.ChainUnaryInterceptor( - tracing.NewGRPCUnaryServerInterceptor(), + metrics.NewUnaryServerInterceptor(), + tracing.NewUnaryServerInterceptor(), ), grpc.ChainStreamInterceptor( - tracing.NewGRPCStreamServerInterceptor(), + metrics.NewStreamServerInterceptor(), + tracing.NewStreamServerInterceptor(), ), } diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 3288083cc..97218da08 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -8,8 +8,9 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -90,10 +91,12 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ec51c6bc6..d132faf6e 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -11,11 +11,13 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -297,10 +299,12 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing_grpc.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing_grpc.NewStreamClientInterceptor(), ), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { -- 2.45.2 From 55c28fd5f439a38e2ae70d5e12d91dc0f04ec3e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:30:46 +0300 Subject: [PATCH 084/233] [#412] cache: Pass DialOptions to create new conn Signed-off-by: Dmitrii Stepanov --- pkg/network/cache/multi.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b8a0aa4bc..ba81df16c 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -10,8 +10,11 @@ import ( rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -72,6 +75,17 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address prmInit.SetResponseInfoCallback(x.opts.ResponseCallback) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), + ), + ) + c.Init(prmInit) err := c.Dial(ctx, prmDial) if err != nil { -- 2.45.2 From 2dd3fc8b7e7c4e00d1b808488f31a4afd94d5516 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 1 Jun 2023 12:56:51 +0300 Subject: [PATCH 085/233] [#xx] cli: Add policy-playground command Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 178 ++++++++++++++++++ cmd/frostfs-cli/modules/container/root.go | 2 + 2 files changed, 180 insertions(+) create mode 100644 cmd/frostfs-cli/modules/container/policy_playground.go diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go new file mode 100644 index 000000000..fa10dc10e --- /dev/null +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -0,0 +1,178 @@ +package container + +import ( + "bufio" + "encoding/hex" + "fmt" + "io" + "os" + "strings" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type policyPlaygroundREPL struct { + cmd *cobra.Command + args []string + nodes map[string]netmap.NodeInfo +} + +func newPolicyPlaygroundREPL(cmd *cobra.Command, args []string) (*policyPlaygroundREPL, error) { + return &policyPlaygroundREPL{ + cmd: cmd, + args: args, + nodes: map[string]netmap.NodeInfo{}, + }, nil +} + +func (repl *policyPlaygroundREPL) handleLs(args []string) error { + if len(args) > 0 { + return fmt.Errorf("too many arguments for command 'ls': got %d, want 0", len(args)) + } + i := 1 + for id, node := range repl.nodes { + var attrs []string + node.IterateAttributes(func(k, v string) { + attrs = append(attrs, fmt.Sprintf("%s:%s", k, v)) + }) + fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) + i++ + } + return nil +} + +func (repl *policyPlaygroundREPL) handleAdd(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'add': got %d, want >0", len(args)) + } + id := args[0] + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + node := repl.nodes[id] + node.SetPublicKey(key) + for _, attr := range args[1:] { + kv := strings.Split(attr, ":") + if len(kv) != 2 { + return fmt.Errorf("node attributes must be in the format 'KEY:VALUE': got %q", attr) + } + node.SetAttribute(kv[0], kv[1]) + } + repl.nodes[id] = node + return nil +} + +func (repl *policyPlaygroundREPL) handleRemove(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'remove': got %d, want >0", len(args)) + } + id := args[0] + if _, exists := repl.nodes[id]; exists { + delete(repl.nodes, id) + return nil + } + return fmt.Errorf("node not found: id=%q", id) +} + +func (repl *policyPlaygroundREPL) handleEval(args []string) error { + policyStr := strings.Join(args, " ") + placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) + if err != nil { + return fmt.Errorf("parsing placement policy: %v", err) + } + nm := repl.netMap() + nodes, err := nm.ContainerNodes(*placementPolicy, nil) + if err != nil { + return fmt.Errorf("building container nodes: %v", err) + } + for i, ns := range nodes { + var ids []string + for _, node := range ns { + ids = append(ids, hex.EncodeToString(node.PublicKey())) + } + fmt.Printf("\t%2d: %v\n", i+1, ids) + } + + return nil +} + +func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { + var nm netmap.NetMap + var nodes []netmap.NodeInfo + for _, node := range repl.nodes { + nodes = append(nodes, node) + } + nm.SetNodes(nodes) + return nm +} + +func (repl *policyPlaygroundREPL) run() error { + if len(viper.GetString(commonflags.RPC)) > 0 { + key := key.GetOrGenerate(repl.cmd) + cli := internalclient.GetSDKClientByFlag(repl.cmd, key, commonflags.RPC) + + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + resp, err := internalclient.NetMapSnapshot(prm) + commonCmd.ExitOnErr(repl.cmd, "unable to get netmap snapshot to populate initial netmap: %w", err) + + for _, node := range resp.NetMap().Nodes() { + id := hex.EncodeToString(node.PublicKey()) + repl.nodes[id] = node + } + } + + cmdHandlers := map[string]func([]string) error{ + "ls": repl.handleLs, + "add": repl.handleAdd, + "remove": repl.handleRemove, + "eval": repl.handleEval, + } + for reader := bufio.NewReader(os.Stdin); ; { + fmt.Print("> ") + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("reading line: %v", err) + } + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + cmd := parts[0] + handler, exists := cmdHandlers[cmd] + if exists { + if err := handler(parts[1:]); err != nil { + fmt.Printf("error: %v\n", err) + } + } else { + fmt.Printf("error: unknown command %q\n", cmd) + } + } +} + +var policyPlaygroundCmd = &cobra.Command{ + Use: "policy-playground", + Short: "A REPL for testing placement policies", + Long: `A REPL for testing placement policies. +If a wallet and endpoint is provided, the initial netmap data will be loaded from the snapshot of the node. Otherwise, an empty playground is created.`, + Run: func(cmd *cobra.Command, args []string) { + repl, err := newPolicyPlaygroundREPL(cmd, args) + commonCmd.ExitOnErr(cmd, "could not create policy playground: %w", err) + commonCmd.ExitOnErr(cmd, "policy playground failed: %w", repl.run()) + }, +} + +func initContainerPolicyPlaygroundCmd() { + commonflags.Init(policyPlaygroundCmd) +} diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index 30a82954a..ab7f5fb90 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -28,6 +28,7 @@ func init() { getExtendedACLCmd, setExtendedACLCmd, containerNodesCmd, + policyPlaygroundCmd, } Cmd.AddCommand(containerChildCommand...) @@ -40,6 +41,7 @@ func init() { initContainerGetEACLCmd() initContainerSetEACLCmd() initContainerNodesCmd() + initContainerPolicyPlaygroundCmd() for _, containerCommand := range containerChildCommand { commonflags.InitAPI(containerCommand) -- 2.45.2 From 63473d0806691c3f7cdd84a98710611338d637c6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Jun 2023 17:21:03 +0300 Subject: [PATCH 086/233] [#417] go.mod: Update neo-go Update to master, because after the API update notary signer is being incorrectly formed. Signed-off-by: Evgenii Stratonikov --- go.mod | 8 ++++++-- go.sum | Bin 98963 -> 99981 bytes pkg/morph/client/constructor.go | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6c49c02b4..d5b3826b2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.25.0 - github.com/nspcc-dev/neo-go v0.101.1 + github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.7.4 github.com/paulmach/orb v0.9.2 @@ -49,6 +49,8 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -72,6 +74,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect @@ -81,7 +84,7 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -114,6 +117,7 @@ require ( google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) retract ( diff --git a/go.sum b/go.sum index b9b3fde4125d22a606b0cca07094d61b19dfcee4..242614781d37fd8f7048774cfe8aef35e58d9d49 100644 GIT binary patch delta 1386 zcmaLXyN=^j0LJld1VaN7+Cf{5R-!=B#P>J`rtoB9$4MN=iJim-X?1+RpTw6Wb~D-? zQLcn!Eh`$H0V(ZDh>`+cfd@cK4+SFG-6DvY?Ogr**LRLi{msdwJwiu_i{Gd)Vi2=!rv#{M#!gP&LY<0Mr8dVeZ{wo)f|tBku~i~hBk*i948XKSA@bn zwwr)7yfD*pnZrXh)5H#PoE8+d8d&p)=%NJWvCzYUfU#(AmS`Habc|LW!}T;4-oRh| zzxodOC>`L}@{nv$<%Vlz!0FmL*B$c{-{*~>;<<@B!ZR0b%?g#={f@r;0HqFDmZPQV zq#Wk{N@4hNuy8G}5t4o~vXynr8^zSsiD*mk9$P=Te6)M@<)@E&Sz+w=j6q^$=WDEq zH_@XGJ@CdUr#7T0^q4uQxN#QY^p#AiW1lDJm?)Jr=c2SX3+tcfAJ;D*zqfn%*n7xD zHb~$aENLq(`6u=5^C$ef%FDh;1IC||y0C^BKB0mH(P@G7_0DQ; zUCqX3G|33nHY)ANJsaI;9wqB)v5-P5@98e+U_=FwN^}>F%q6Cd2E)pc6}us2*HZpw zl;2*Q)XNti?*9Jn`#)~d%xwA)$>U|@6p48_Pqg{^C8LN1Cwnc1btKSMHfTSl^9Dbo z2f8Qut)R*}m68t(ab4|i$=%FRB+C<-GOoIjfeDb%TyhO(49h>z`$c5)>;}vx+aV*8 K%-CZuPyYo=$I#~h delta 326 zcmeC}Wt-f}w&9(@<}jnR+y*8J8HQHrMn!oBWsXh({{9iUQK4DdWl6cA1qL|*`4O(Z zr5Pawo;js{1`$4%p^=mO>{K>~+bJteZb;90q~FV=q`06Wyxcr1J1@e(%uPEi*tgib%rn!kAksZ8EF{Uuvm(sdz`!&z zbn=FoCX=7fXmU3)G%_$UFgGse%G v=~dwEUmO^o>f@j2UX)^D>EY*?Uu04060YqCGEV}RS(E?I2-xg1Yt3r_2 Date: Wed, 24 May 2023 16:51:57 +0300 Subject: [PATCH 087/233] [#406] cli: Pass context to internal client Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 80 +++++++++---------- cmd/frostfs-cli/modules/accounting/balance.go | 2 +- cmd/frostfs-cli/modules/container/create.go | 8 +- cmd/frostfs-cli/modules/container/delete.go | 8 +- cmd/frostfs-cli/modules/container/get.go | 2 +- cmd/frostfs-cli/modules/container/get_eacl.go | 2 +- cmd/frostfs-cli/modules/container/list.go | 4 +- .../modules/container/list_objects.go | 4 +- cmd/frostfs-cli/modules/container/nodes.go | 2 +- cmd/frostfs-cli/modules/container/set_eacl.go | 6 +- cmd/frostfs-cli/modules/netmap/get_epoch.go | 2 +- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/snapshot.go | 2 +- cmd/frostfs-cli/modules/object/delete.go | 2 +- cmd/frostfs-cli/modules/object/get.go | 2 +- cmd/frostfs-cli/modules/object/hash.go | 4 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-cli/modules/object/lock.go | 2 +- cmd/frostfs-cli/modules/object/put.go | 2 +- cmd/frostfs-cli/modules/object/range.go | 2 +- cmd/frostfs-cli/modules/object/search.go | 2 +- cmd/frostfs-cli/modules/object/util.go | 12 +-- cmd/frostfs-cli/modules/session/create.go | 9 ++- 24 files changed, 83 insertions(+), 82 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 875ccf904..0e0aadbb3 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -37,8 +37,8 @@ func (x BalanceOfRes) Balance() accounting.Decimal { // BalanceOf requests the current balance of a FrostFS user. // // Returns any error which prevented the operation from completing correctly in error return. -func BalanceOf(prm BalanceOfPrm) (res BalanceOfRes, err error) { - res.cliRes, err = prm.cli.BalanceGet(context.Background(), prm.PrmBalanceGet) +func BalanceOf(ctx context.Context, prm BalanceOfPrm) (res BalanceOfRes, err error) { + res.cliRes, err = prm.cli.BalanceGet(ctx, prm.PrmBalanceGet) return } @@ -62,8 +62,8 @@ func (x ListContainersRes) IDList() []cid.ID { // ListContainers requests a list of FrostFS user's containers. // // Returns any error which prevented the operation from completing correctly in error return. -func ListContainers(prm ListContainersPrm) (res ListContainersRes, err error) { - res.cliRes, err = prm.cli.ContainerList(context.Background(), prm.PrmContainerList) +func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContainersRes, err error) { + res.cliRes, err = prm.cli.ContainerList(ctx, prm.PrmContainerList) return } @@ -92,8 +92,8 @@ func (x PutContainerRes) ID() cid.ID { // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func PutContainer(prm PutContainerPrm) (res PutContainerRes, err error) { - cliRes, err := prm.cli.ContainerPut(context.Background(), prm.PrmContainerPut) +func PutContainer(ctx context.Context, prm PutContainerPrm) (res PutContainerRes, err error) { + cliRes, err := prm.cli.ContainerPut(ctx, prm.PrmContainerPut) if err == nil { res.cnr = cliRes.ID() } @@ -125,20 +125,20 @@ func (x GetContainerRes) Container() containerSDK.Container { // GetContainer reads a container from FrostFS by ID. // // Returns any error which prevented the operation from completing correctly in error return. -func GetContainer(prm GetContainerPrm) (res GetContainerRes, err error) { - res.cliRes, err = prm.cli.ContainerGet(context.Background(), prm.cliPrm) +func GetContainer(ctx context.Context, prm GetContainerPrm) (res GetContainerRes, err error) { + res.cliRes, err = prm.cli.ContainerGet(ctx, prm.cliPrm) return } // IsACLExtendable checks if ACL of the container referenced by the given identifier // can be extended. Client connection MUST BE correctly established in advance. -func IsACLExtendable(c *client.Client, cnr cid.ID) (bool, error) { +func IsACLExtendable(ctx context.Context, c *client.Client, cnr cid.ID) (bool, error) { var prm GetContainerPrm prm.SetClient(c) prm.SetContainer(cnr) - res, err := GetContainer(prm) + res, err := GetContainer(ctx, prm) if err != nil { return false, fmt.Errorf("get container from the FrostFS: %w", err) } @@ -163,8 +163,8 @@ type DeleteContainerRes struct{} // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteContainer(prm DeleteContainerPrm) (res DeleteContainerRes, err error) { - _, err = prm.cli.ContainerDelete(context.Background(), prm.PrmContainerDelete) +func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteContainerRes, err error) { + _, err = prm.cli.ContainerDelete(ctx, prm.PrmContainerDelete) return } @@ -188,8 +188,8 @@ func (x EACLRes) EACL() eacl.Table { // EACL reads eACL table from FrostFS by container ID. // // Returns any error which prevented the operation from completing correctly in error return. -func EACL(prm EACLPrm) (res EACLRes, err error) { - res.cliRes, err = prm.cli.ContainerEACL(context.Background(), prm.PrmContainerEACL) +func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { + res.cliRes, err = prm.cli.ContainerEACL(ctx, prm.PrmContainerEACL) return } @@ -211,8 +211,8 @@ type SetEACLRes struct{} // Success can be verified by reading by container identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func SetEACL(prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.cli.ContainerSetEACL(context.Background(), prm.PrmContainerSetEACL) +func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { + _, err = prm.cli.ContainerSetEACL(ctx, prm.PrmContainerSetEACL) return } @@ -236,8 +236,8 @@ func (x NetworkInfoRes) NetworkInfo() netmap.NetworkInfo { // NetworkInfo reads information about the FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func NetworkInfo(prm NetworkInfoPrm) (res NetworkInfoRes, err error) { - res.cliRes, err = prm.cli.NetworkInfo(context.Background(), prm.PrmNetworkInfo) +func NetworkInfo(ctx context.Context, prm NetworkInfoPrm) (res NetworkInfoRes, err error) { + res.cliRes, err = prm.cli.NetworkInfo(ctx, prm.PrmNetworkInfo) return } @@ -266,8 +266,8 @@ func (x NodeInfoRes) LatestVersion() version.Version { // NodeInfo requests information about the remote server from FrostFS netmap. // // Returns any error which prevented the operation from completing correctly in error return. -func NodeInfo(prm NodeInfoPrm) (res NodeInfoRes, err error) { - res.cliRes, err = prm.cli.EndpointInfo(context.Background(), prm.PrmEndpointInfo) +func NodeInfo(ctx context.Context, prm NodeInfoPrm) (res NodeInfoRes, err error) { + res.cliRes, err = prm.cli.EndpointInfo(ctx, prm.PrmEndpointInfo) return } @@ -290,8 +290,8 @@ func (x NetMapSnapshotRes) NetMap() netmap.NetMap { // NetMapSnapshot requests current network view of the remote server. // // Returns any error which prevented the operation from completing correctly in error return. -func NetMapSnapshot(prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { - res.cliRes, err = prm.cli.NetMapSnapshot(context.Background(), client.PrmNetMapSnapshot{}) +func NetMapSnapshot(ctx context.Context, prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { + res.cliRes, err = prm.cli.NetMapSnapshot(ctx, client.PrmNetMapSnapshot{}) return } @@ -319,8 +319,8 @@ func (x CreateSessionRes) SessionKey() []byte { // CreateSession opens a new unlimited session with the remote node. // // Returns any error which prevented the operation from completing correctly in error return. -func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { - res.cliRes, err = prm.cli.SessionCreate(context.Background(), prm.PrmSessionCreate) +func CreateSession(ctx context.Context, prm CreateSessionPrm) (res CreateSessionRes, err error) { + res.cliRes, err = prm.cli.SessionCreate(ctx, prm.PrmSessionCreate) return } @@ -373,7 +373,7 @@ func (x PutObjectRes) ID() oid.ID { // PutObject saves the object in FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { +func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { var putPrm client.PrmObjectPutInit if prm.sessionToken != nil { @@ -391,7 +391,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { putPrm.WithXHeaders(prm.xHeaders...) putPrm.SetCopiesNumberByVectors(prm.copyNum) - wrt, err := prm.cli.ObjectPutInit(context.Background(), putPrm) + wrt, err := prm.cli.ObjectPutInit(ctx, putPrm) if err != nil { return nil, fmt.Errorf("init object writing: %w", err) } @@ -471,7 +471,7 @@ func (x DeleteObjectRes) Tombstone() oid.ID { // DeleteObject marks an object to be removed from FrostFS through tombstone placement. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { +func DeleteObject(ctx context.Context, prm DeleteObjectPrm) (*DeleteObjectRes, error) { var delPrm client.PrmObjectDelete delPrm.FromContainer(prm.objAddr.Container()) delPrm.ByID(prm.objAddr.Object()) @@ -486,7 +486,7 @@ func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { delPrm.WithXHeaders(prm.xHeaders...) - cliRes, err := prm.cli.ObjectDelete(context.Background(), delPrm) + cliRes, err := prm.cli.ObjectDelete(ctx, delPrm) if err != nil { return nil, fmt.Errorf("remove object via client: %w", err) } @@ -527,7 +527,7 @@ func (x GetObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { +func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { var getPrm client.PrmObjectGet getPrm.FromContainer(prm.objAddr.Container()) getPrm.ByID(prm.objAddr.Object()) @@ -550,7 +550,7 @@ func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { getPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectGetInit(context.Background(), getPrm) + rdr, err := prm.cli.ObjectGetInit(ctx, getPrm) if err != nil { return nil, fmt.Errorf("init object reading on client: %w", err) } @@ -603,7 +603,7 @@ func (x HeadObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { +func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { var cliPrm client.PrmObjectHead cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -626,7 +626,7 @@ func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHead(context.Background(), cliPrm) + res, err := prm.cli.ObjectHead(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read object header via client: %w", err) } @@ -668,7 +668,7 @@ func (x SearchObjectsRes) IDList() []oid.ID { // SearchObjects selects objects from the container which match the filters. // // Returns any error which prevented the operation from completing correctly in error return. -func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { +func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { var cliPrm client.PrmObjectSearch cliPrm.InContainer(prm.cnrID) cliPrm.SetFilters(prm.filters) @@ -687,7 +687,7 @@ func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectSearchInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectSearchInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init object search: %w", err) } @@ -758,7 +758,7 @@ func (x HashPayloadRangesRes) HashList() [][]byte { // // Returns any error which prevented the operation from completing correctly in error return. // Returns an error if number of received hashes differs with the number of requested ranges. -func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { +func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { var cliPrm client.PrmObjectHash cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -792,7 +792,7 @@ func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHash(context.Background(), cliPrm) + res, err := prm.cli.ObjectHash(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read payload hashes via client: %w", err) } @@ -826,7 +826,7 @@ type PayloadRangeRes struct{} // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { +func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { var cliPrm client.PrmObjectRange cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -852,7 +852,7 @@ func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectRangeInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } @@ -886,12 +886,12 @@ type SyncContainerRes struct{} // Interrupts on any writer error. // // Panics if a container passed as a parameter is nil. -func SyncContainerSettings(prm SyncContainerPrm) (*SyncContainerRes, error) { +func SyncContainerSettings(ctx context.Context, prm SyncContainerPrm) (*SyncContainerRes, error) { if prm.c == nil { panic("sync container settings with the network: nil container") } - err := client.SyncContainerWithNetwork(context.Background(), prm.c, prm.cli) + err := client.SyncContainerWithNetwork(ctx, prm.c, prm.cli) if err != nil { return nil, err } diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index bec40f1ff..5ed8f9403 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -41,7 +41,7 @@ var accountingBalanceCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.BalanceOf(prm) + res, err := internalclient.BalanceOf(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) // print to stdout diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 873ef3235..2acdfc16f 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -48,7 +48,7 @@ It will be stored in sidechain when inner ring will accepts it.`, var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot to validate container placement, "+ "use --force option to skip this check: %w", err) @@ -96,7 +96,7 @@ It will be stored in sidechain when inner ring will accepts it.`, syncContainerPrm.SetClient(cli) syncContainerPrm.SetContainer(&cnr) - _, err = internalclient.SyncContainerSettings(syncContainerPrm) + _, err = internalclient.SyncContainerSettings(cmd.Context(), syncContainerPrm) commonCmd.ExitOnErr(cmd, "syncing container's settings rpc error: %w", err) var putPrm internalclient.PutContainerPrm @@ -107,7 +107,7 @@ It will be stored in sidechain when inner ring will accepts it.`, putPrm.WithinSession(*tok) } - res, err := internalclient.PutContainer(putPrm) + res, err := internalclient.PutContainer(cmd.Context(), putPrm) commonCmd.ExitOnErr(cmd, "put container rpc error: %w", err) id := res.ID() @@ -124,7 +124,7 @@ It will be stored in sidechain when inner ring will accepts it.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err == nil { cmd.Println("container has been persisted on sidechain") return diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 308c7b942..54b49a32d 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -34,7 +34,7 @@ Only owner of the container has a permission to remove container.`, getPrm.SetClient(cli) getPrm.SetContainer(id) - resGet, err := internalclient.GetContainer(getPrm) + resGet, err := internalclient.GetContainer(cmd.Context(), getPrm) commonCmd.ExitOnErr(cmd, "can't get the container: %w", err) owner := resGet.Container().Owner() @@ -72,7 +72,7 @@ Only owner of the container has a permission to remove container.`, common.PrintVerbose(cmd, "Searching for LOCK objects...") - res, err := internalclient.SearchObjects(searchPrm) + res, err := internalclient.SearchObjects(cmd.Context(), searchPrm) commonCmd.ExitOnErr(cmd, "can't search for LOCK objects: %w", err) if len(res.IDList()) != 0 { @@ -91,7 +91,7 @@ Only owner of the container has a permission to remove container.`, delPrm.WithinSession(*tok) } - _, err := internalclient.DeleteContainer(delPrm) + _, err := internalclient.DeleteContainer(cmd.Context(), delPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cmd.Println("container delete method invoked") @@ -106,7 +106,7 @@ Only owner of the container has a permission to remove container.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err != nil { cmd.Println("container has been removed:", containerID) return diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 2db1f7c8d..90bcc190a 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -151,7 +151,7 @@ func getContainer(cmd *cobra.Command) (container.Container, *ecdsa.PrivateKey) { prm.SetClient(cli) prm.SetContainer(id) - res, err := internalclient.GetContainer(prm) + res, err := internalclient.GetContainer(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cnr = res.Container() diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index bc04a84fc..21ea5b5bc 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -24,7 +24,7 @@ var getExtendedACLCmd = &cobra.Command{ eaclPrm.SetClient(cli) eaclPrm.SetContainer(id) - res, err := internalclient.EACL(eaclPrm) + res, err := internalclient.EACL(cmd.Context(), eaclPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) eaclTable := res.EACL() diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 33dd17943..1c9e4767f 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -49,7 +49,7 @@ var listContainersCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.ListContainers(prm) + res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var prmGet internalclient.GetContainerPrm @@ -63,7 +63,7 @@ var listContainersCmd = &cobra.Command{ } prmGet.SetContainer(cnrID) - res, err := internalclient.GetContainer(prmGet) + res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { cmd.Printf(" failed to read attributes: %v\n", err) continue diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index aef4a1f80..e417560e8 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -51,7 +51,7 @@ var listContainerObjectsCmd = &cobra.Command{ prmSearch.SetContainerID(id) prmSearch.SetFilters(*filters) - res, err := internalclient.SearchObjects(prmSearch) + res, err := internalclient.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) objectIDs := res.IDList() @@ -65,7 +65,7 @@ var listContainerObjectsCmd = &cobra.Command{ addr.SetObject(objectIDs[i]) prmHead.SetAddress(addr) - resHead, err := internalclient.HeadObject(prmHead) + resHead, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { attrs := resHead.Header().Attributes() for i := range attrs { diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index d89772fcc..8b0f266a7 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -31,7 +31,7 @@ var containerNodesCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot", err) var id cid.ID diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index 0b781589f..c88d5767b 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -38,7 +38,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, if !flagVarsSetEACL.noPreCheck { cmd.Println("Checking the ability to modify access rights in the container...") - extendable, err := internalclient.IsACLExtendable(cli, id) + extendable, err := internalclient.IsACLExtendable(cmd.Context(), cli, id) commonCmd.ExitOnErr(cmd, "Extensibility check failure: %w", err) if !extendable { @@ -56,7 +56,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, setEACLPrm.WithinSession(*tok) } - _, err := internalclient.SetEACL(setEACLPrm) + _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) if containerAwait { @@ -72,7 +72,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - res, err := internalclient.EACL(getEACLPrm) + res, err := internalclient.EACL(cmd.Context(), getEACLPrm) if err == nil { // compare binary values because EACL could have been set already table := res.EACL() diff --git a/cmd/frostfs-cli/modules/netmap/get_epoch.go b/cmd/frostfs-cli/modules/netmap/get_epoch.go index 6e05721ff..a9c2e1f19 100644 --- a/cmd/frostfs-cli/modules/netmap/get_epoch.go +++ b/cmd/frostfs-cli/modules/netmap/get_epoch.go @@ -19,7 +19,7 @@ var getEpochCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index 17acfd59c..c6ab0b6f6 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -23,7 +23,7 @@ var netInfoCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 4a94d9e70..3b2113efb 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -25,7 +25,7 @@ var nodeInfoCmd = &cobra.Command{ var prm internalclient.NodeInfoPrm prm.SetClient(cli) - res, err := internalclient.NodeInfo(prm) + res, err := internalclient.NodeInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) prettyPrintNodeInfo(cmd, res.NodeInfo()) diff --git a/cmd/frostfs-cli/modules/netmap/snapshot.go b/cmd/frostfs-cli/modules/netmap/snapshot.go index 0878f5ceb..eaaf598b9 100644 --- a/cmd/frostfs-cli/modules/netmap/snapshot.go +++ b/cmd/frostfs-cli/modules/netmap/snapshot.go @@ -19,7 +19,7 @@ var snapshotCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - res, err := internalclient.NetMapSnapshot(prm) + res, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) commonCmd.PrettyPrintNetMap(cmd, res.NetMap(), false) diff --git a/cmd/frostfs-cli/modules/object/delete.go b/cmd/frostfs-cli/modules/object/delete.go index 25d5703de..e4e9cddb8 100644 --- a/cmd/frostfs-cli/modules/object/delete.go +++ b/cmd/frostfs-cli/modules/object/delete.go @@ -65,7 +65,7 @@ func deleteObject(cmd *cobra.Command, _ []string) { Prepare(cmd, &prm) prm.SetAddress(objAddr) - res, err := internalclient.DeleteObject(prm) + res, err := internalclient.DeleteObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) tomb := res.Tombstone() diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index 68e47da6f..c72a26239 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -90,7 +90,7 @@ func getObject(cmd *cobra.Command, _ []string) { }) } - res, err := internalclient.GetObject(prm) + res, err := internalclient.GetObject(cmd.Context(), prm) if p != nil { p.Finish() } diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index c7d734e67..26243e7e7 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -75,7 +75,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { headPrm.SetAddress(objAddr) // get hash of full payload through HEAD (may be user can do it through dedicated command?) - res, err := internalclient.HeadObject(headPrm) + res, err := internalclient.HeadObject(cmd.Context(), headPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var cs checksum.Checksum @@ -108,7 +108,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { hashPrm.TZ() } - res, err := internalclient.HashPayloadRanges(hashPrm) + res, err := internalclient.HashPayloadRanges(cmd.Context(), hashPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) hs := res.HashList() diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 139563e24..6fce04490 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -64,7 +64,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { prm.SetAddress(objAddr) prm.SetMainOnlyFlag(mainOnly) - res, err := internalclient.HeadObject(prm) + res, err := internalclient.HeadObject(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index e6fbedd2c..fa1898586 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -104,7 +104,7 @@ var objectLockCmd = &cobra.Command{ Prepare(cmd, &prm) prm.SetHeader(obj) - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "Store lock object in FrostFS: %w", err) cmd.Printf("Lock object ID: %s\n", res.ID()) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 6fd91ca4b..af9e9fd0b 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -131,7 +131,7 @@ func putObject(cmd *cobra.Command, _ []string) { prm.SetCopiesNumberByVectors(cn) } - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(cmd.Context(), prm) if p != nil { p.Finish() } diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index a594204f0..76425d948 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -87,7 +87,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { prm.SetRange(ranges[0]) prm.SetPayloadWriter(out) - _, err = internalclient.PayloadRange(prm) + _, err = internalclient.PayloadRange(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return diff --git a/cmd/frostfs-cli/modules/object/search.go b/cmd/frostfs-cli/modules/object/search.go index de4a8a3b4..b603e5fe8 100644 --- a/cmd/frostfs-cli/modules/object/search.go +++ b/cmd/frostfs-cli/modules/object/search.go @@ -61,7 +61,7 @@ func searchObject(cmd *cobra.Command, _ []string) { prm.SetContainerID(cnr) prm.SetFilters(sf) - res, err := internalclient.SearchObjects(prm) + res, err := internalclient.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) ids := res.IDList() diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 204409df3..094b62314 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -278,7 +278,7 @@ func OpenSessionViaClient(cmd *cobra.Command, dst SessionPrm, cli *client.Client common.PrintVerbose(cmd, "Opening remote session with the node...") - err := sessionCli.CreateSession(&tok, cli, sessionLifetime) + err := sessionCli.CreateSession(cmd.Context(), &tok, cli, sessionLifetime) commonCmd.ExitOnErr(cmd, "open remote session: %w", err) common.PrintVerbose(cmd, "Session successfully opened.") @@ -354,7 +354,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, Prepare(cmd, &prmHead) - _, err := internal.HeadObject(prmHead) + _, err := internal.HeadObject(cmd.Context(), prmHead) var errSplit *object.SplitInfoError @@ -396,7 +396,7 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl prmHead.SetRawFlag(false) // client is already set - res, err := internal.HeadObject(prmHead) + res, err := internal.HeadObject(cmd.Context(), prmHead) if err == nil { children := res.Header().Children() @@ -425,7 +425,7 @@ func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo prm.SetClient(cli) prm.SetFilters(query) - res, err := internal.SearchObjects(prm) + res, err := internal.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "failed to search objects by split ID: %w", err) parts := res.IDList() @@ -463,7 +463,7 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p addrObj.SetObject(idMember) prmHead.SetAddress(addrObj) - res, err = internal.HeadObject(prmHead) + res, err = internal.HeadObject(cmd.Context(), prmHead) commonCmd.ExitOnErr(cmd, "failed to read split chain member's header: %w", err) idMember, ok = res.Header().PreviousID() @@ -490,7 +490,7 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p prmSearch.SetContainerID(cnr) prmSearch.SetFilters(query) - resSearch, err := internal.SearchObjects(prmSearch) + resSearch, err := internal.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "failed to find object children: %w", err) list := resSearch.IDList() diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 341681f5b..0cc12606e 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -1,6 +1,7 @@ package session import ( + "context" "fmt" "os" @@ -64,7 +65,7 @@ func createSession(cmd *cobra.Command, _ []string) { var tok session.Object - err = CreateSession(&tok, c, lifetime) + err = CreateSession(cmd.Context(), &tok, c, lifetime) commonCmd.ExitOnErr(cmd, "can't create session: %w", err) var data []byte @@ -86,11 +87,11 @@ func createSession(cmd *cobra.Command, _ []string) { // number of epochs. // // Fills ID, lifetime and session key. -func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error { +func CreateSession(ctx context.Context, dst *session.Object, c *client.Client, lifetime uint64) error { var netInfoPrm internalclient.NetworkInfoPrm netInfoPrm.SetClient(c) - ni, err := internalclient.NetworkInfo(netInfoPrm) + ni, err := internalclient.NetworkInfo(ctx, netInfoPrm) if err != nil { return fmt.Errorf("can't fetch network info: %w", err) } @@ -102,7 +103,7 @@ func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error sessionPrm.SetClient(c) sessionPrm.SetExp(exp) - sessionRes, err := internalclient.CreateSession(sessionPrm) + sessionRes, err := internalclient.CreateSession(ctx, sessionPrm) if err != nil { return fmt.Errorf("can't open session: %w", err) } -- 2.45.2 From 96c98435913a8208c323e81c7e4ac7f9e9601868 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Jun 2023 16:30:54 +0300 Subject: [PATCH 088/233] [#406] cmd/common: Execute PersistentPostRun() on errors Signed-off-by: Evgenii Stratonikov --- cmd/internal/common/exit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index 9b912ddc0..d38f27e02 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -46,5 +46,6 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) + cmd.PersistentPostRun(cmd, nil) os.Exit(code) } -- 2.45.2 From f1f56ef8712cfe37af377c86e491d3388ec7500e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 May 2023 14:50:19 +0300 Subject: [PATCH 089/233] [#406] cli: Add --trace flag Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/sdk.go | 5 ++ cmd/frostfs-cli/internal/common/tracing.go | 64 +++++++++++++++++++ cmd/frostfs-cli/internal/commonflags/flags.go | 5 ++ cmd/frostfs-cli/modules/accounting/root.go | 3 + cmd/frostfs-cli/modules/container/create.go | 1 + cmd/frostfs-cli/modules/container/delete.go | 1 + cmd/frostfs-cli/modules/container/root.go | 3 + cmd/frostfs-cli/modules/netmap/root.go | 3 + cmd/frostfs-cli/modules/object/root.go | 3 + cmd/frostfs-cli/modules/session/create.go | 5 +- 10 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-cli/internal/common/tracing.go diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 13dacc04c..79d3dcb0d 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -12,9 +12,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" "github.com/spf13/viper" + "google.golang.org/grpc" ) var errInvalidEndpoint = errors.New("provided RPC endpoint is incorrect") @@ -59,6 +61,9 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey common.PrintVerbose(cmd, "Set request timeout to %s.", timeout) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor())) c.Init(prmInit) diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go new file mode 100644 index 000000000..7a9684667 --- /dev/null +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -0,0 +1,64 @@ +package common + +import ( + "context" + "fmt" + "sort" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/spf13/cobra" + "go.opentelemetry.io/otel/trace" +) + +type spanKey struct{} + +// StopClientCommandSpan stops tracing span for the command and prints trace ID on the standard output. +func StopClientCommandSpan(cmd *cobra.Command, _ []string) { + span, ok := cmd.Context().Value(spanKey{}).(trace.Span) + if !ok { + return + } + + span.End() + + // Noop provider cannot fail on flush. + _ = tracing.Shutdown(cmd.Context()) + + cmd.PrintErrf("Trace ID: %s\n", span.SpanContext().TraceID()) +} + +// StartClientCommandSpan starts tracing span for the command. +func StartClientCommandSpan(cmd *cobra.Command) { + enableTracing, err := cmd.Flags().GetBool(commonflags.TracingFlag) + if err != nil || !enableTracing { + return + } + + _, err = tracing.Setup(cmd.Context(), tracing.Config{ + Enabled: true, + Exporter: tracing.NoOpExporter, + Service: "frostfs-cli", + Version: misc.Version, + }) + commonCmd.ExitOnErr(cmd, "init tracing: %w", err) + + var components sort.StringSlice + for c := cmd; c != nil; c = c.Parent() { + fmt.Println(c.Name()) + components = append(components, c.Name()) + } + for i, j := 0, len(components)-1; i < j; { + components.Swap(i, j) + i++ + j-- + } + + operation := strings.Join(components, ".") + ctx, span := tracing.StartSpanFromContext(cmd.Context(), operation) + ctx = context.WithValue(ctx, spanKey{}, span) + cmd.SetContext(ctx) +} diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 810e62107..5049dc3b1 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -47,6 +47,9 @@ const ( OIDFlag = "oid" OIDFlagUsage = "Object ID." + + TracingFlag = "trace" + TracingFlagUsage = "Generate trace ID and print it." ) // Init adds common flags to the command: @@ -54,12 +57,14 @@ const ( // - WalletPath, // - Account, // - RPC, +// - Tracing, // - Timeout. func Init(cmd *cobra.Command) { InitWithoutRPC(cmd) ff := cmd.Flags() ff.StringP(RPC, RPCShorthand, RPCDefault, RPCUsage) + ff.Bool(TracingFlag, false, TracingFlagUsage) ff.DurationP(Timeout, TimeoutShorthand, TimeoutDefault, TimeoutUsage) } diff --git a/cmd/frostfs-cli/modules/accounting/root.go b/cmd/frostfs-cli/modules/accounting/root.go index 8ab8aa125..f94488b6f 100644 --- a/cmd/frostfs-cli/modules/accounting/root.go +++ b/cmd/frostfs-cli/modules/accounting/root.go @@ -1,6 +1,7 @@ package accounting import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -17,7 +18,9 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.WalletPath, flags.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, flags.Lookup(commonflags.Account)) _ = viper.BindPFlag(commonflags.RPC, flags.Lookup(commonflags.RPC)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 2acdfc16f..e5bbeacd4 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -141,6 +141,7 @@ func initContainerCreateCmd() { // Init common flags flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage) flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 54b49a32d..f5b69edfd 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -124,6 +124,7 @@ func initContainerDeleteCmd() { flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) flags.BoolVar(&containerAwait, "await", false, "Block execution until container is removed") diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index ab7f5fb90..f3c3e0e3a 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/netmap/root.go b/cmd/frostfs-cli/modules/netmap/root.go index aaa83f12f..006ac6d9f 100644 --- a/cmd/frostfs-cli/modules/netmap/root.go +++ b/cmd/frostfs-cli/modules/netmap/root.go @@ -1,6 +1,7 @@ package netmap import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -14,7 +15,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 886153075..23badc576 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -1,6 +1,7 @@ package object import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 0cc12606e..53f6e8bc4 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -6,6 +6,7 @@ import ( "os" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -29,10 +30,12 @@ var createCmd = &cobra.Command{ Use: "create", Short: "Create session token", Run: createSession, - PersistentPreRun: func(cmd *cobra.Command, _ []string) { + PersistentPreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, cmd.Flags().Lookup(commonflags.Account)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { -- 2.45.2 From f8c1e0639d93af814793580dc9f50c0f1dd3a321 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 14:45:30 +0300 Subject: [PATCH 090/233] [#422] cli: Provide context to NetmapSnapshot() Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index fa10dc10e..65255f9c2 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -121,7 +121,7 @@ func (repl *policyPlaygroundREPL) run() error { var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resp, err := internalclient.NetMapSnapshot(prm) + resp, err := internalclient.NetMapSnapshot(repl.cmd.Context(), prm) commonCmd.ExitOnErr(repl.cmd, "unable to get netmap snapshot to populate initial netmap: %w", err) for _, node := range resp.NetMap().Nodes() { -- 2.45.2 From a770b89fd8fe726078fa9f20d5a35e22f844b2b5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 14:48:07 +0300 Subject: [PATCH 091/233] [#422] adm: Fix ValidatorCount problems after neo-go update Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_test.go | 2 +- cmd/frostfs-adm/internal/modules/morph/local_client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 07d2da8cc..30a7168dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -119,7 +119,7 @@ func generateTestData(t *testing.T, dir string, size int) error { cfg := config.Config{} cfg.ProtocolConfiguration.Magic = 12345 - cfg.ProtocolConfiguration.ValidatorsCount = size + cfg.ProtocolConfiguration.ValidatorsCount = uint32(size) cfg.ProtocolConfiguration.TimePerBlock = time.Second cfg.ProtocolConfiguration.StandbyCommittee = pubs // sorted by glagolic letters cfg.ProtocolConfiguration.P2PSigExtensions = true diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 816f9da4c..45d09c387 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -62,7 +62,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet return nil, err } - m := smartcontract.GetDefaultHonestNodeCount(cfg.ProtocolConfiguration.ValidatorsCount) + m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) accounts := make([]*wallet.Account, len(wallets)) for i := range accounts { accounts[i], err = getWalletAccount(wallets[i], consensusAccountName) -- 2.45.2 From 189a367ef24304260372e6201038ad2e06bdb049 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 5 Jun 2023 18:54:59 +0300 Subject: [PATCH 092/233] [#390] frostfs-cli: Pass bearer token to Tree srv * Add --bearer flag for "tree" subcommand Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 3 ++- cmd/frostfs-cli/modules/tree/add_by_path.go | 2 +- cmd/frostfs-cli/modules/tree/get_by_path.go | 2 +- cmd/frostfs-cli/modules/tree/root.go | 4 ++++ cmd/internal/common/exit.go | 4 +++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 707a4d8ee..ea9245271 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -60,7 +61,7 @@ func add(cmd *cobra.Command, _ []string) { TreeId: tid, ParentId: pid, Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index e83408a89..6b182e1e4 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -72,7 +72,7 @@ func addByPath(cmd *cobra.Command, _ []string) { // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 75acbaedf..6f6fc7ff9 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -71,7 +71,7 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index c70e6f5e2..b5de0dd6f 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -32,6 +32,8 @@ const ( pathAttributeFlagKey = "pattr" latestOnlyFlagKey = "latest" + + bearerFlagKey = "bearer" ) func initCTID(cmd *cobra.Command) { @@ -42,4 +44,6 @@ func initCTID(cmd *cobra.Command) { ff.String(treeIDFlagKey, "", "Tree ID") _ = cmd.MarkFlagRequired(treeIDFlagKey) + + ff.StringP(bearerFlagKey, "", "", "Path to bearer token") } diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index d38f27e02..9e4fa3098 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -46,6 +46,8 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) - cmd.PersistentPostRun(cmd, nil) + if cmd.PersistentPostRun != nil { + cmd.PersistentPostRun(cmd, nil) + } os.Exit(code) } -- 2.45.2 From 263c6fdc50693e5df0b453c593e2a9a7808a7909 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Jun 2023 17:28:04 +0300 Subject: [PATCH 093/233] [#372] node: Add metrics for the error counter in the engine Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/engine.go | 2 ++ pkg/local_object_storage/engine/metrics.go | 3 +++ pkg/local_object_storage/engine/shards.go | 15 ++++++++++++ .../shard/metrics_test.go | 13 ++++++++++ pkg/local_object_storage/shard/shard.go | 24 +++++++++++++++++++ pkg/metrics/engine.go | 14 +++++++++++ 6 files changed, 71 insertions(+) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 21e863005..7c36811e7 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -132,6 +132,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er } errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) } @@ -150,6 +151,7 @@ func (e *StorageEngine) reportShardError( } errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) } diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 1be888eae..f9e9191cd 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -26,6 +26,9 @@ type MetricRegister interface { AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteErrorCounter(shardID string) WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 07d22d3fe..c4c356a75 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -62,6 +62,18 @@ func (m *metricsWithID) AddToPayloadSize(size int64) { m.mw.AddToPayloadCounter(m.id, size) } +func (m *metricsWithID) IncErrorCounter() { + m.mw.IncErrorCounter(m.id) +} + +func (m *metricsWithID) ClearErrorCounter() { + m.mw.ClearErrorCounter(m.id) +} + +func (m *metricsWithID) DeleteErrorCounter() { + m.mw.DeleteErrorCounter(m.id) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. @@ -174,6 +186,8 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } + sh.DeleteErrorCounter() + ss = append(ss, sh) delete(e.shards, id) @@ -281,6 +295,7 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte if id.String() == shID { if resetErrorCounter { sh.errorCount.Store(0) + sh.Shard.ClearErrorCounter() } return sh.SetMode(m) } diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 16f6989c4..f1581b6d4 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -23,6 +23,7 @@ type metricsStore struct { cnrSize map[string]int64 pldSize int64 readOnly bool + errCounter int64 } func (m metricsStore) SetShardID(_ string) {} @@ -68,6 +69,18 @@ func (m *metricsStore) AddToPayloadSize(size int64) { m.pldSize += size } +func (m *metricsStore) IncErrorCounter() { + m.errCounter += 1 +} + +func (m *metricsStore) ClearErrorCounter() { + m.errCounter = 0 +} + +func (m *metricsStore) DeleteErrorCounter() { + m.errCounter = 0 +} + const physical = "phy" const logical = "logic" diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 2123bca1f..b740fc572 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -75,6 +75,12 @@ type MetricsWriter interface { SetShardID(id string) // SetReadonly must set shard readonly state. SetReadonly(readonly bool) + // IncErrorCounter increment error counter. + IncErrorCounter() + // ClearErrorCounter clear error counter. + ClearErrorCounter() + // DeleteErrorCounter delete error counter. + DeleteErrorCounter() } type cfg struct { @@ -428,3 +434,21 @@ func (s *Shard) addToPayloadSize(size int64) { s.cfg.metricsWriter.AddToPayloadSize(size) } } + +func (s *Shard) IncErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.IncErrorCounter() + } +} + +func (s *Shard) ClearErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.ClearErrorCounter() + } +} + +func (s *Shard) DeleteErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.DeleteErrorCounter() + } +} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4e78f4ac2..7992da9f8 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -24,6 +24,7 @@ type ( listObjectsDuration prometheus.Counter containerSize *prometheus.GaugeVec payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec } ) @@ -44,6 +45,7 @@ func newEngineMetrics() engineMetrics { listObjectsDuration: newEngineMethodDurationCounter("list_objects"), containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), } } @@ -123,3 +125,15 @@ func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } + +func (m engineMetrics) IncErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() +} + +func (m engineMetrics) ClearErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) +} + +func (m engineMetrics) DeleteErrorCounter(shardID string) { + m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) +} -- 2.45.2 From 5b75432ca2a74dc518f93bbc5655dbb5d1e44eea Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 09:35:19 +0300 Subject: [PATCH 094/233] [#431] metrics: Add missed label Signed-off-by: Dmitrii Stepanov --- pkg/metrics/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 3c56aa2ba..a20c3ca6a 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -176,7 +176,7 @@ func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess}) + }, []string{wcShardID, wcSuccess, wcStorage}) } func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { -- 2.45.2 From 508e2064ebfb2e3b7dce2921a7f0791f2757129c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 14:08:28 +0300 Subject: [PATCH 095/233] [#438] cli: Drop tracing debug print Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/common/tracing.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go index 7a9684667..30c2f2b1a 100644 --- a/cmd/frostfs-cli/internal/common/tracing.go +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -2,7 +2,6 @@ package common import ( "context" - "fmt" "sort" "strings" @@ -48,7 +47,6 @@ func StartClientCommandSpan(cmd *cobra.Command) { var components sort.StringSlice for c := cmd; c != nil; c = c.Parent() { - fmt.Println(c.Name()) components = append(components, c.Name()) } for i, j := 0, len(components)-1; i < j; { -- 2.45.2 From 9bc1a25c0766ca04ef0d02161bcafebe7a403f67 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 17:10:25 +0300 Subject: [PATCH 096/233] [#434] metrics: Fix writecache duration Signed-off-by: Dmitrii Stepanov --- pkg/metrics/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index a20c3ca6a..67b9d8caf 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -167,7 +167,7 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b wcSuccess: fmt.Sprintf("%v", success), wcStorage: storageType, }, - ).Observe(float64(d)) + ).Observe(d.Seconds()) } func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { -- 2.45.2 From cd92d8a9e74556b53060fcf097bfd87ab96707bc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 15:36:05 +0300 Subject: [PATCH 097/233] [#423] go.mod: Update sdk-go and hrw Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 8 ++++---- go.sum | Bin 99981 -> 99899 bytes 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83ec8e41a..6afb8f700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Changelog for FrostFS Node - `go.opentelemetry.io/otel` to `v1.15.1` - `go.opentelemetry.io/otel/trace` to `v1.15.1` - `github.com/spf13/cast` to `v1.5.1` +- `git.frostfs.info/TrueCloudLab/hrw` to `v1.2.1` ### Updating from v0.36.0 diff --git a/go.mod b/go.mod index d5b3826b2..8bb31686e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b - git.frostfs.info/TrueCloudLab/hrw v1.2.0 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 + git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 @@ -44,7 +44,7 @@ require ( require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -98,7 +98,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/twmb/murmur3 v1.1.7 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.13 // indirect go.mongodb.org/mongo-driver v1.11.6 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect diff --git a/go.sum b/go.sum index 242614781d37fd8f7048774cfe8aef35e58d9d49..c64597f658464a44b5bbfe0e3f710392c6cd6e30 100644 GIT binary patch delta 513 zcmZwDyKd7^0LF2#RH9Ocmd+(Y3c55Yt52w)*3hCaamr%&JA-NDz72cOwvo&f}eG(~~FW6?v$bZvwPrsu;~ zwh-YPZO2{ncxw8W_8ZZGDZ855xm!sbr#4q+S_*U%O{4b1(Tn!^@e}D`4fnD^lGz2Z2C>1?^CgQd zG)h&{Wtv7qO-+-{%$UlAxBqr__6eYF6Vi)eVNCJmgp2{I`Me38o#lILjhHL>O#&k; V*0&fiWUsyd{`3*r_kX;-{|)mMrkMZ$ delta 550 zcma)&zmAh|0LSBUi6#zrE-rU=(8Neue*Y-+HYhDef%aOY6yo4f`lASx(%w;yIQRhG zA}`@+($U1k=^jEK!qv@VFuB3O2l#xxf4+V^xVbs_!XNfH3?SwpLQM#WfGG7AKK2?* zcv==4LzXmAR~9`Lm{-NHn8b?B4%V*6qKSRIvv@)0Vm;heA0EG~o*zEZ|7~M~w3VcO zEd?tSjGZX!2H`A{%Lq(#%PP`7DrS5~TL_#dhRRf!OR;b#ZB*}%9s*EO4bJEC33$#EJ0ffQbSFlY8#!b$L)Q12TUI zPv=**SrA%cNfbsK5qjelqrqMrVrjB7s_!Q+_Mc9Ee+GPA&C+ctGyqw#vvi0Z!yL3- hv_6yjI%SN7Ivra)bk8@rX9rb$`|JU~|8e{F+aIsvsmA~S -- 2.45.2 From 41ab4d070e6b700491ae741cf4d9ffbb0845105f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 15:39:16 +0300 Subject: [PATCH 098/233] [#423] *: Use hrw.StringHash() where possible Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- pkg/local_object_storage/engine/engine_test.go | 2 +- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/remove_copies.go | 2 +- pkg/local_object_storage/engine/shards.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index af976f977..d99e9aa89 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -216,7 +216,7 @@ func addressHash(addr *oid.Address, path string) uint64 { a = addr.EncodeToString() } - return hrw.Hash([]byte(a + path)) + return hrw.StringHash(a + path) } // converts uint64 to hex string. diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 9c239739f..11c0848fe 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -101,7 +101,7 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard errorCount: new(atomic.Uint32), Shard: s, }, - hash: hrw.Hash([]byte(s.ID().String())), + hash: hrw.StringHash(s.ID().String()), } te.engine.shardPools[s.ID().String()] = pool te.shardIDs = append(te.shardIDs, s.ID()) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index c103ede73..73b7a7830 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -373,7 +373,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) + hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(addr.EncodeToString())) for j := range shards { select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 1ea569928..7681e0e50 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -110,7 +110,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address } for addr := range ch { - h := hrw.Hash([]byte(addr.EncodeToString())) + h := hrw.StringHash(addr.EncodeToString()) shards := sortShardsByWeight(shards, h) found := false for i := range shards { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index c4c356a75..fd9605e9b 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -162,7 +162,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { errorCount: new(atomic.Uint32), Shard: sh, }, - hash: hrw.Hash([]byte(strID)), + hash: hrw.StringHash(strID), } e.shardPools[strID] = pool @@ -237,7 +237,7 @@ func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() s e.mtx.RLock() defer e.mtx.RUnlock() - h := hrw.Hash([]byte(objAddr.EncodeToString())) + h := hrw.StringHash(objAddr.EncodeToString()) shards := make([]hashedShard, 0, len(e.shards)) for _, sh := range e.shards { shards = append(shards, hashedShard(sh)) -- 2.45.2 From 0400153b7d9974b80bb1d57f4f4ed243ffb6d4cd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Jun 2023 17:33:51 +0300 Subject: [PATCH 099/233] [#424] object: Do not store large slices in pool Dynamically growing an unbounded buffers can cause a large amount of memory to be pinned and never be freed. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/pool.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/put/pool.go index 5726856e5..ebe214caf 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/put/pool.go @@ -4,7 +4,10 @@ import ( "sync" ) -const defaultAllocSize = 1024 +const ( + defaultAllocSize = 1024 + poolSliceMaxSize = 128 * 1024 +) type payload struct { Data []byte @@ -19,6 +22,9 @@ func getPayload() *payload { } func putPayload(p *payload) { + if cap(p.Data) > poolSliceMaxSize { + return + } p.Data = p.Data[:0] putBytesPool.Put(p) } -- 2.45.2 From 83d600ed77cbabd9db925e025b3eaae5de92b46a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 18:08:32 +0300 Subject: [PATCH 100/233] [#424] node: Update api-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | Bin 99899 -> 99899 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8bb31686e..0fdcb4409 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 diff --git a/go.sum b/go.sum index c64597f658464a44b5bbfe0e3f710392c6cd6e30..5da5a9a9ed7db66bf1919e0c23dc30541c154708 100644 GIT binary patch delta 114 zcmdnp!?wGJZG$zhia}pp`nR^iJ7jMQEEzxnMIn3d1A6chM`rWnSNBVsZ(lJT25|y yX>dk)Qcg~~SBh(DMxL)_dT~a!PmxcOcc6!J=;V4{Co${>HDBl5ew~*wr2+s4tRy@D -- 2.45.2 From 0c40d98f7a7ad6f39755c4c0a7e298e419d2aafd Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 9 Jun 2023 11:47:39 +0300 Subject: [PATCH 101/233] [#375] Add log metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-ir/main.go | 1 + cmd/frostfs-node/config.go | 2 ++ pkg/util/logger/logger.go | 8 ++++++++ pkg/util/logger/metrics.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 pkg/util/logger/metrics.go diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 1718a46ab..81fcaa489 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -61,6 +61,7 @@ func main() { cfg, err = newConfig() exitErr(err) + logPrm.MetricsNamespace = "frostfs-ir" err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f8605c21e..e4547364d 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -542,6 +542,8 @@ func initCfg(appCfg *config.Config) *cfg { logPrm, err := c.loggerPrm() fatalOnErr(err) + logPrm.MetricsNamespace = "frostfs-node" + log, err := logger.NewLogger(logPrm) fatalOnErr(err) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 4a536368a..fcac09321 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -31,6 +31,9 @@ type Prm struct { // support runtime rereading level zapcore.Level + // MetricsNamespace is the namespace string used for log counter metrics + MetricsNamespace string + // do not support runtime rereading } @@ -79,10 +82,15 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) + m := newLogMetrics(prm.MetricsNamespace) + c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) { + m.Inc(e.Level, sd == zapcore.LogDropped) + } lZap, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go new file mode 100644 index 000000000..10be4a8f0 --- /dev/null +++ b/pkg/util/logger/metrics.go @@ -0,0 +1,37 @@ +package logger + +import ( + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap/zapcore" +) + +const ( + logSubsystem = "log" + logLevelLabel = "level" + logDroppedLabel = "dropped" +) + +type logMetrics struct { + logCount *prometheus.CounterVec +} + +func newLogMetrics(namespace string) *logMetrics { + return &logMetrics{ + logCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: logSubsystem, + Name: "log_count", + Help: "Total log entries emitted or dropped by severity level", + }, []string{logLevelLabel, logDroppedLabel}), + } +} + +func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { + m.logCount.With(prometheus.Labels{ + logLevelLabel: level.String(), + logDroppedLabel: strconv.FormatBool(dropped), + }).Inc() +} -- 2.45.2 From 2541d319de64d6b111830993188bdd4b597e39d4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:26:59 +0300 Subject: [PATCH 102/233] [#266] pilorama: Allow to get current tree height Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree.go | 16 +++++++++ pkg/local_object_storage/pilorama/boltdb.go | 34 +++++++++++++++++++ pkg/local_object_storage/pilorama/forest.go | 9 +++++ .../pilorama/forest_test.go | 13 +++++-- .../pilorama/interface.go | 2 ++ pkg/local_object_storage/shard/tree.go | 16 +++++++++ 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 6b8f83f31..08c6d26b0 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -311,6 +311,22 @@ func (e *StorageEngine) TreeExists(ctx context.Context, cid cidSDK.ID, treeID st return err == nil, err } +func (e *StorageEngine) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cid, treeID) + if err != nil { + return 0, nil + } + return lst[index].TreeHeight(ctx, cid, treeID) +} + // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeUpdateLastSyncHeight", diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9b62f0649..5b2c97f10 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -188,6 +188,40 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin }) } +func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return 0, ErrDegradedMode + } + + var height uint64 + var retErr error + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cid, treeID)) + if treeRoot != nil { + k, _ := treeRoot.Bucket(logBucket).Cursor().Last() + height = binary.BigEndian.Uint64(k) + } else { + retErr = ErrTreeNotFound + } + return nil + }) + if err == nil { + err = retErr + } + return height, err +} + // TreeExists implements the Forest interface. func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 672a38edd..7c5897ed0 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -214,6 +214,15 @@ func (f *memoryForest) TreeList(_ context.Context, cid cid.ID) ([]string, error) return res, nil } +func (f *memoryForest) TreeHeight(_ context.Context, cid cid.ID, treeID string) (uint64, error) { + fullID := cid.EncodeToString() + "/" + treeID + tree, ok := f.treeMap[fullID] + if !ok { + return 0, ErrTreeNotFound + } + return tree.operations[len(tree.operations)-1].Time, nil +} + // TreeExists implements the pilorama.Forest interface. func (f *memoryForest) TreeExists(_ context.Context, cid cid.ID, treeID string) (bool, error) { fullID := cid.EncodeToString() + "/" + treeID diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index ebb4667f5..9e8e98863 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -604,10 +604,19 @@ func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...O checkExists(t, false, cid, treeID) }) - require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Parent: 0, Child: 1}, false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Meta: Meta{Time: 11}, Parent: 0, Child: 1}, false)) checkExists(t, true, cid, treeID) + + height, err := s.TreeHeight(context.Background(), cid, treeID) + require.NoError(t, err) + require.EqualValues(t, 11, height) + checkExists(t, false, cidtest.ID(), treeID) // different CID, same tree - checkExists(t, false, cid, "another tree") // same CID, different tree + + _, err = s.TreeHeight(context.Background(), cidtest.ID(), treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + + checkExists(t, false, cid, "another tree") // same CID, different tree t.Run("can be removed", func(t *testing.T) { require.NoError(t, s.TreeDrop(context.Background(), cid, treeID)) diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 9ca721be8..c8287c6d4 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -50,6 +50,8 @@ type Forest interface { TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error // TreeLastSyncHeight returns last log height synchronized with _all_ container nodes. TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) + // TreeHeight returns current tree height. + TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) } type ForestStorage interface { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index ad89fa633..2331d37a8 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -255,6 +255,22 @@ func (s *Shard) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { return s.pilorama.TreeList(ctx, cid) } +func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeHeight", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + if s.pilorama == nil { + return 0, ErrPiloramaDisabled + } + return s.pilorama.TreeHeight(ctx, cid, treeID) +} + // TreeExists implements the pilorama.Forest interface. func (s *Shard) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeExists", -- 2.45.2 From e69a1e84827a1839e6c1d63955985aeb24eee985 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:32:01 +0300 Subject: [PATCH 103/233] [#266] services/tree: Return operation log up to some height Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/service.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 96e547f36..4364095e2 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -532,9 +532,13 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } h := b.GetHeight() + lastHeight, err := s.forest.TreeHeight(srv.Context(), cid, b.GetTreeId()) + if err != nil { + return err + } for { lm, err := s.forest.TreeGetOpLog(srv.Context(), cid, b.GetTreeId(), h) - if err != nil || lm.Time == 0 { + if err != nil || lm.Time == 0 || lastHeight < lm.Time { return err } -- 2.45.2 From 957a43a12408fd9e4751d8b72d29bd08f4103ac0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:43:25 +0300 Subject: [PATCH 104/233] [#266] services/tree: Add sync check Do not accept requests until initial sync is finished. `Apply` is deliberately left out -- we don't want to miss anything new. Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/service.go | 39 ++++++++++++++++++++++++++++++++++++ pkg/services/tree/sync.go | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 4364095e2..12d970c42 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -31,6 +32,8 @@ type Service struct { syncChan chan struct{} syncPool *ants.Pool + initialSyncDone atomic.Bool + // cnrMap contains existing (used) container IDs. cnrMap map[cidSDK.ID]struct{} // cnrMapMtx protects cnrMap @@ -90,6 +93,10 @@ func (s *Service) Shutdown() { } func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -138,6 +145,10 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error } func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -198,6 +209,10 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP } func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -247,6 +262,10 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon // Move applies client operation to the specified tree and pushes in queue // for replication on other nodes. func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -295,6 +314,10 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er } func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -371,6 +394,10 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -500,6 +527,10 @@ func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, e } func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -560,6 +591,10 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + var cid cidSDK.ID err := cid.Decode(req.GetBody().GetContainerId()) @@ -643,5 +678,9 @@ func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeI } func (s *Service) Healthcheck(context.Context, *HealthcheckRequest) (*HealthcheckResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + return new(HealthcheckResponse), nil } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index d132faf6e..e44e8dbbf 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -388,7 +388,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) span.End() - continue + break } newMap, cnrsToSync := s.containersToSync(cnrs) @@ -402,6 +402,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.metrics.AddSyncDuration(time.Since(start), true) span.End() } + s.initialSyncDone.Store(true) } } -- 2.45.2 From 898f0686b1197caf60e57cd95074ac9cc73a2239 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:06:04 +0300 Subject: [PATCH 105/233] [#409] node: Log maintenance state on startup Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index c6623a385..9d8ad6a9a 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -267,6 +267,8 @@ func initNetmapState(c *cfg) { stateWord = "online" case ni.IsOffline(): stateWord = "offline" + case ni.IsMaintenance(): + stateWord = "maintenance" } } -- 2.45.2 From e68384d4e3b494e50b1c82990c312a5fa4af923e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:08:56 +0300 Subject: [PATCH 106/233] [#409] node: Fetch last bootstrap info on startup Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 64 ++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 9d8ad6a9a..c86e3a2e7 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -256,21 +256,10 @@ func initNetmapState(c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) - ni, err := c.netmapLocalNodeState(epoch) + ni, err := c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) - stateWord := "undefined" - - if ni != nil { - switch { - case ni.IsOnline(): - stateWord = "online" - case ni.IsOffline(): - stateWord = "offline" - case ni.IsMaintenance(): - stateWord = "maintenance" - } - } + stateWord := nodeState(ni) c.log.Info(logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), @@ -282,6 +271,55 @@ func initNetmapState(c *cfg) { c.handleLocalNodeInfo(ni) } +func nodeState(ni *netmapSDK.NodeInfo) string { + if ni != nil { + switch { + case ni.IsOnline(): + return "online" + case ni.IsOffline(): + return "offline" + case ni.IsMaintenance(): + return "maintenance" + } + } + return "undefined" +} + +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { + nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() + if err != nil { + return nil, err + } + + var candidate *netmapSDK.NodeInfo + for i := range nmNodes { + if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { + candidate = &nmNodes[i] + break + } + } + + node, err := c.netmapLocalNodeState(epoch) + if err != nil { + return nil, err + } + + if candidate == nil { + return node, nil + } + + nmState := nodeState(node) + candidateState := nodeState(candidate) + if nmState != candidateState { + // This happens when the node was switched to maintenance without epoch tick. + // We expect it to continue staying in maintenance. + c.log.Info("candidate status is different from the netmap status, the former takes priority", + zap.String("netmap", nmState), + zap.String("candidate", candidateState)) + } + return candidate, nil +} + func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { // calculate current network state nm, err := c.cfgNetmap.wrapper.GetNetMapByEpoch(epoch) -- 2.45.2 From 4f83ab0fb43d969e1f8b81aa555b7894678aef88 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:11:59 +0300 Subject: [PATCH 107/233] [#409] node: Do not sent initial bootstrap under maintenance Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 8 ++++++++ internal/logs/logs.go | 1 + 2 files changed, 9 insertions(+) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index c86e3a2e7..f1ea8b40e 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -225,6 +225,10 @@ func addNewEpochNotificationHandlers(c *cfg) { // Must be called after initNetmapService. func bootstrapNode(c *cfg) { if c.needBootstrap() { + if c.IsMaintenance() { + c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + return + } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -266,6 +270,10 @@ func initNetmapState(c *cfg) { zap.String("state", stateWord), ) + if ni != nil && ni.IsMaintenance() { + c.isMaintenance.Store(true) + } + c.cfgNetmap.state.setCurrentEpoch(epoch) c.cfgNetmap.startEpoch = epoch c.handleLocalNodeInfo(ni) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9b800adfc..dc54f2d2a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -491,4 +491,5 @@ const ( EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" + FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" ) -- 2.45.2 From 90e9247b69955a04a1e3aa97b042d52bdfb5b03b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 26 May 2023 12:15:50 +0300 Subject: [PATCH 108/233] [#371] ir: Add morph cache metrics Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/innerring/initialization.go | 11 ++--- pkg/innerring/innerring.go | 14 +++--- pkg/metrics/innerring.go | 18 +++++--- pkg/metrics/morphcache.go | 77 +++++++++++++++++++++++++++++++++ pkg/metrics/node.go | 3 +- pkg/metrics/treeservice.go | 8 ++++ pkg/morph/client/client.go | 3 ++ pkg/morph/client/constructor.go | 14 +++++- pkg/morph/client/nns.go | 18 ++++++++ pkg/morph/client/notary.go | 9 ++++ pkg/services/tree/metrics.go | 6 --- 11 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 pkg/metrics/morphcache.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index c49d22509..05e503f2f 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -469,11 +469,12 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, - cfg: cfg, - key: s.key, - name: morphPrefix, - from: fromSideChainBlock, + log: s.log, + cfg: cfg, + key: s.key, + name: morphPrefix, + from: fromSideChainBlock, + morphCacheMetric: s.metrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index cbcb4699a..335d3d179 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -102,12 +102,13 @@ type ( } chainParams struct { - log *logger.Logger - cfg *viper.Viper - key *keys.PrivateKey - name string - sgn *transaction.Signer - from uint32 // block height + log *logger.Logger + cfg *viper.Viper + key *keys.PrivateKey + name string + sgn *transaction.Signer + from uint32 // block height + morphCacheMetric metrics.MorphCacheMetrics } ) @@ -465,6 +466,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c errChan <- fmt.Errorf("%s chain connection has been lost", p.name) }), client.WithSwitchInterval(p.cfg.GetDuration(p.name+".switch_interval")), + client.WithMorphCacheMetrics(p.morphCacheMetric), ) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 9d8b76bf9..d756a539f 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -16,9 +16,10 @@ const ( // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch prometheus.Gauge - health prometheus.Gauge - eventDuration *prometheus.HistogramVec + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec + morphCacheMetrics *morphCacheMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -45,9 +46,10 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { ) return &InnerRingServiceMetrics{ - epoch: epoch, - health: health, - eventDuration: eventDuration, + epoch: epoch, + health: health, + eventDuration: eventDuration, + morphCacheMetrics: newMorphCacheMetrics(), } } @@ -67,3 +69,7 @@ func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success b innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } + +func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { + return m.morphCacheMetrics +} diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go new file mode 100644 index 000000000..7f6f2d7d5 --- /dev/null +++ b/pkg/metrics/morphcache.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + mcSubsystem = "morphcache" + mcSuccess = "success" +) + +type MorphCacheMetrics interface { + AddNNSContractHashDuration(success bool, d time.Duration) + + AddGroupKeyDuration(success bool, d time.Duration) + + AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) +} + +type morphCacheMetrics struct { + // Duration of processing get nns contract hash request + nnsContractHashDuration *prometheus.HistogramVec + + // Duration of processing get group key request + groupKeyDuration *prometheus.HistogramVec + + // Duration of processing get tx height request + txHeightDuration *prometheus.HistogramVec +} + +var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) + +func newMorphCacheMetrics() *morphCacheMetrics { + return &morphCacheMetrics{ + nnsContractHashDuration: newMCMethodDurationCounter("nns_contract_hash"), + groupKeyDuration: newMCMethodDurationCounter("group_key"), + txHeightDuration: newMCMethodDurationCounter("tx_height"), + } +} + +func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { + m.nnsContractHashDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { + m.groupKeyDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { + m.txHeightDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func newMCMethodDurationCounter(method string) *prometheus.HistogramVec { + return metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: mcSubsystem, + Name: fmt.Sprintf("%s_req_duration_seconds", method), + Help: fmt.Sprintf("Accumulated %s request process duration", method), + }, []string{mcSuccess}) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 8819ba15b..16a4c2b5b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,7 +1,6 @@ package metrics import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -67,7 +66,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { return m.writeCacheMetrics } -func (m *NodeMetrics) TreeService() tree.MetricsRegister { +func (m *NodeMetrics) TreeService() TreeMetricsRegister { return m.treeService } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 903ef3496..48a450c76 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -10,12 +10,20 @@ import ( const treeServiceLabelSuccess = "success" +type TreeMetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type treeServiceMetrics struct { replicateTaskDuration *prometheus.HistogramVec replicateWaitDuration *prometheus.HistogramVec syncOpDuration *prometheus.HistogramVec } +var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) + func newTreeServiceMetrics() *treeServiceMetrics { const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 6a7a5b51a..f03ee1dbf 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -10,6 +10,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -88,6 +89,8 @@ type cache struct { nnsHash *util.Uint160 gKey *keys.PublicKey txHeights *lru.Cache[util.Uint256, uint32] + + metrics metrics.MorphCacheMetrics } func (c *cache) nns() *util.Uint160 { diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 1d55db71c..8b5fb3ff0 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -42,6 +43,8 @@ type cfg struct { inactiveModeCb Callback switchInterval time.Duration + + morphCacheMetrics metrics.MorphCacheMetrics } const ( @@ -104,7 +107,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), + cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, acc: acc, accAddr: accAddr, @@ -195,10 +198,11 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto }}) } -func newClientCache() cache { +func newClientCache(morphCacheMetrics metrics.MorphCacheMetrics) cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ txHeights: c, + metrics: morphCacheMetrics, } } @@ -282,3 +286,9 @@ func WithSwitchInterval(i time.Duration) Option { c.switchInterval = i } } + +func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { + return func(c *cfg) { + c.morphCacheMetrics = morphCacheMetrics + } +} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 2f7079dfe..d8d798b07 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -81,6 +82,13 @@ func (c *Client) NNSHash() (util.Uint160, error) { return util.Uint160{}, ErrConnectionLost } + success := false + startedAt := time.Now() + + defer func() { + c.cache.metrics.AddNNSContractHashDuration(success, time.Since(startedAt)) + }() + nnsHash := c.cache.nns() if nnsHash == nil { @@ -92,6 +100,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { c.cache.setNNSHash(cs.Hash) nnsHash = &cs.Hash } + success = true return *nnsHash, nil } @@ -221,7 +230,14 @@ func (c *Client) SetGroupSignerScope() error { // contractGroupKey returns public key designating FrostFS contract group. func (c *Client) contractGroupKey() (*keys.PublicKey, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddGroupKeyDuration(success, time.Since(startedAt)) + }() + if gKey := c.cache.groupKey(); gKey != nil { + success = true return gKey, nil } @@ -251,5 +267,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { } c.cache.setGroupKey(pub) + + success = true return pub, nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 1ed1ca912..3a3ff0b46 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -8,6 +8,7 @@ import ( "math" "math/big" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" @@ -789,7 +790,14 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) } func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddTxHeightDuration(h, success, time.Since(startedAt)) + }() + if rh, ok := c.cache.txHeights.Get(h); ok { + success = true return rh, nil } height, err := c.client.GetTransactionHeight(h) @@ -797,5 +805,6 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { return 0, err } c.cache.txHeights.Add(h, height) + success = true return height, nil } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 0f0e4ee57..53708c4f1 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,12 +2,6 @@ package tree import "time" -type MetricsRegister interface { - AddReplicateTaskDuration(time.Duration, bool) - AddReplicateWaitDuration(time.Duration, bool) - AddSyncDuration(time.Duration, bool) -} - type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} -- 2.45.2 From 4887f489a195418340992eca63d609b293c26e6f Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 8 Jun 2023 16:37:46 +0300 Subject: [PATCH 109/233] [#17] Add morph client metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/morph.go | 2 + pkg/innerring/initialization.go | 14 ++--- pkg/innerring/innerring.go | 6 +-- pkg/innerring/state.go | 8 +-- pkg/metrics/morph.go | 82 ++++++++++++++++++++++++++++++ pkg/morph/client/client.go | 30 ++++++++++- pkg/morph/client/constructor.go | 20 ++++++++ pkg/morph/client/notary.go | 7 +++ pkg/morph/metrics/metrics.go | 17 +++++++ pkg/morph/subscriber/subscriber.go | 5 ++ pkg/services/tree/metrics.go | 6 +++ 11 files changed, 182 insertions(+), 15 deletions(-) create mode 100644 pkg/metrics/morph.go create mode 100644 pkg/morph/metrics/metrics.go diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2e086f994..ae50b8a8a 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -9,6 +9,7 @@ import ( morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -43,6 +44,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), + client.WithMetrics(metrics.NewMorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { c.internalErr <- errors.New("morph connection has been lost") diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 05e503f2f..52ffb10a1 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -52,7 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -163,7 +163,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli // create governance processor governanceProcessor, err := governance.New(&governance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, FrostFSClient: frostfsCli, NetmapClient: s.netmapClient, AlphabetState: s, @@ -233,7 +233,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -258,7 +258,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, @@ -277,7 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -298,7 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, @@ -474,7 +474,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- key: s.key, name: morphPrefix, from: fromSideChainBlock, - morphCacheMetric: s.metrics.MorphCacheMetrics(), + morphCacheMetric: s.irMetrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 335d3d179..1567e40d3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -58,7 +58,7 @@ type ( persistate *state.PersistentStorage // metrics - metrics *metrics.InnerRingServiceMetrics + irMetrics *metrics.InnerRingServiceMetrics // notary configuration feeConfig *config.FeeConfig @@ -328,8 +328,8 @@ func (s *Server) registerStarter(f func() error) { func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error server := &Server{ - log: log, - metrics: metrics.NewInnerRingMetrics(), + log: log, + irMetrics: metrics.NewInnerRingMetrics(), } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 27f265ae2..c5adb71eb 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -29,8 +29,8 @@ func (s *Server) EpochCounter() uint64 { // epoch counter. func (s *Server) SetEpochCounter(val uint64) { s.epochCounter.Store(val) - if s.metrics != nil { - s.metrics.SetEpoch(val) + if s.irMetrics != nil { + s.irMetrics.SetEpoch(val) } } @@ -154,8 +154,8 @@ func (s *Server) ResetEpochTimer(h uint32) error { func (s *Server) setHealthStatus(hs control.HealthStatus) { s.healthStatus.Store(int32(hs)) - if s.metrics != nil { - s.metrics.SetHealth(int32(hs)) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(hs)) } } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go new file mode 100644 index 000000000..cc851d3db --- /dev/null +++ b/pkg/metrics/morph.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "strconv" + "time" + + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + morphSubsystem = "morph" + morphNotificationTypeLabel = "notification_type" + morphInvokeTypeLabel = "invoke_type" + morphContractLabel = "contract" + morphMethodLabel = "method" + morphSuccessLabel = "success" +) + +type morphClientMetrics struct { + switchCount prometheus.Counter + lastBlock prometheus.Gauge + notificationCount *prometheus.CounterVec + invokeDuration *prometheus.HistogramVec +} + +func NewMorphClientMetrics() morphmetrics.Register { + return &morphClientMetrics{ + switchCount: metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "switch_count", + Help: "Number of endpoint switches", + }), + lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "last_block", + Help: "Index of the last received block", + }), + notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "notification_count", + Help: "Number of notifications received by notification type", + }, []string{morphNotificationTypeLabel}), + invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "invoke_duration_seconds", + Help: "Cummulative duration of contract invocations", + }, []string{morphInvokeTypeLabel, morphContractLabel, morphMethodLabel, morphSuccessLabel}), + } +} + +func (m *morphClientMetrics) IncSwitchCount() { + m.switchCount.Inc() +} + +func (m *morphClientMetrics) SetLastBlock(index uint32) { + m.lastBlock.Set(float64(index)) +} + +func (m *morphClientMetrics) IncNotificationCount(typ string) { + m.notificationCount.With( + prometheus.Labels{ + morphNotificationTypeLabel: typ, + }, + ).Inc() +} + +func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { + m.invokeDuration.With( + prometheus.Labels{ + morphInvokeTypeLabel: typ, + morphContractLabel: contract, + morphMethodLabel: method, + morphSuccessLabel: strconv.FormatBool(success), + }, + ).Observe(d.Seconds()) +} diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index f03ee1dbf..606f3bd66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -48,7 +49,8 @@ import ( type Client struct { cache cache - logger *logger.Logger // logging component + logger *logger.Logger // logging component + metrics morphmetrics.Register client *rpcclient.WSClient // neo-go websocket client rpcActor *actor.Actor // neo-go RPC actor @@ -172,6 +174,12 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Supported args types: int64, string, util.Uint160, []byte and bool. func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("Invoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -189,6 +197,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) + success = true return nil } @@ -196,6 +205,12 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util.Uint160, method string, args ...interface{}) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvokeIterator", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -228,12 +243,20 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util } items, err = c.rpcActor.TraverseIterator(sid, &r, 0) } + + success = err == nil return err } // TestInvoke invokes contract method locally in neo-go node. This method should // be used to read data from smart-contract. func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) (res []stackitem.Item, err error) { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -250,6 +273,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) } + success = true return val.Stack, nil } @@ -512,6 +536,10 @@ func (c *Client) NotificationChannel() <-chan rpcclient.Notification { return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } +func (c *Client) Metrics() morphmetrics.Register { + return c.metrics +} + func (c *Client) setActor(act *actor.Actor) { c.rpcActor = act c.gasToken = nep17.New(act, gas.Hash) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 8b5fb3ff0..6cc66c5cc 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -32,6 +33,8 @@ type cfg struct { logger *logger.Logger // logging component + metrics morphmetrics.Register + waitInterval time.Duration signer *transaction.Signer @@ -60,6 +63,7 @@ func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, logger: &logger.Logger{Logger: zap.L()}, + metrics: morphmetrics.NoopRegister{}, waitInterval: defaultWaitInterval, signer: &transaction.Signer{ Scopes: transaction.Global, @@ -80,6 +84,7 @@ func defaultConfig() *cfg { // - signer with the global scope; // - wait interval: 500ms; // - logger: &logger.Logger{Logger: zap.L()}. +// - metrics: metrics.NoopRegister // // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, @@ -109,6 +114,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er cli := &Client{ cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, + metrics: cfg.metrics, acc: acc, accAddr: accAddr, cfg: *cfg, @@ -235,6 +241,20 @@ func WithLogger(logger *logger.Logger) Option { } } +// WithMetrics returns a client constructor option +// that specifies the component for reporting metrics. +// +// Ignores nil value. +// +// If option not provided, NoopMetrics is used. +func WithMetrics(metrics morphmetrics.Register) Option { + return func(c *cfg) { + if metrics != nil { + c.metrics = metrics + } + } +} + // WithSigner returns a client constructor option // that specifies the signer and the scope of the transaction. // diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3a3ff0b46..680ba6a68 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -444,6 +444,12 @@ func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args } func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("notaryInvoke", contract.String(), method, success, time.Since(start)) + }() + alphabetList, err := c.notary.alphabetSource() if err != nil { return err @@ -485,6 +491,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint zap.String("tx_hash", mainH.StringLE()), zap.String("fallback_hash", fbH.StringLE())) + success = true return nil } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go new file mode 100644 index 000000000..9e41a0b86 --- /dev/null +++ b/pkg/morph/metrics/metrics.go @@ -0,0 +1,17 @@ +package metrics + +import "time" + +type Register interface { + IncSwitchCount() + SetLastBlock(uint32) + IncNotificationCount(notificationType string) + ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) +} + +type NoopRegister struct{} + +func (NoopRegister) IncSwitchCount() {} +func (NoopRegister) SetLastBlock(uint32) {} +func (NoopRegister) IncNotificationCount(string) {} +func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 608872dec..4076111f0 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -200,18 +200,22 @@ routeloop: break routeloop case ev, ok := <-curr.NotifyChan: if ok { + s.client.Metrics().IncNotificationCount("notify") s.notifyChan <- ev } else { connLost = true } case ev, ok := <-curr.BlockChan: if ok { + s.client.Metrics().IncNotificationCount("block") + s.client.Metrics().SetLastBlock(ev.Index) s.blockChan <- ev } else { connLost = true } case ev, ok := <-curr.NotaryChan: if ok { + s.client.Metrics().IncNotificationCount("notary") s.notaryChan <- ev } else { connLost = true @@ -262,6 +266,7 @@ func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) ( s.current = chs s.Unlock() + s.client.Metrics().IncSwitchCount() return true, cliCh } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 53708c4f1..0f0e4ee57 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,6 +2,12 @@ package tree import "time" +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} -- 2.45.2 From 344d6b2ae1f241a43bdeacc88ff2b5772d0da508 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 13 Jun 2023 15:27:38 +0300 Subject: [PATCH 110/233] [#xx] Fix invalid log metric namespace identifier Signed-off-by: Alejandro Lopez --- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-node/config.go | 2 +- pkg/util/logger/metrics.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 81fcaa489..70199b094 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -61,7 +61,7 @@ func main() { cfg, err = newConfig() exitErr(err) - logPrm.MetricsNamespace = "frostfs-ir" + logPrm.MetricsNamespace = "frostfs_ir" err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e4547364d..758803291 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -542,7 +542,7 @@ func initCfg(appCfg *config.Config) *cfg { logPrm, err := c.loggerPrm() fatalOnErr(err) - logPrm.MetricsNamespace = "frostfs-node" + logPrm.MetricsNamespace = "frostfs_node" log, err := logger.NewLogger(logPrm) fatalOnErr(err) diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go index 10be4a8f0..708583473 100644 --- a/pkg/util/logger/metrics.go +++ b/pkg/util/logger/metrics.go @@ -9,7 +9,7 @@ import ( ) const ( - logSubsystem = "log" + logSubsystem = "logger" logLevelLabel = "level" logDroppedLabel = "dropped" ) @@ -23,7 +23,7 @@ func newLogMetrics(namespace string) *logMetrics { logCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: logSubsystem, - Name: "log_count", + Name: "entry_count", Help: "Total log entries emitted or dropped by severity level", }, []string{logLevelLabel, logDroppedLabel}), } -- 2.45.2 From fb8fee0c8edd3cca1d9aaa40bf7334df46400964 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 13 Jun 2023 14:19:33 +0300 Subject: [PATCH 111/233] [#442] Use strconv.FormatBool instead of untyped conversion Signed-off-by: Alejandro Lopez --- pkg/metrics/gc.go | 6 +++--- pkg/metrics/morphcache.go | 7 ++++--- pkg/metrics/treeservice.go | 8 ++++---- pkg/metrics/writecache.go | 5 +++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index 2457c0c6b..9c00d8722 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -1,7 +1,7 @@ package metrics import ( - "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -64,7 +64,7 @@ func newGCMetrics() *gcMetrics { func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { m.runDuration.With(prometheus.Labels{ gcShardID: shardID, - gcSuccess: fmt.Sprintf("%v", success), + gcSuccess: strconv.FormatBool(success), }).Add(d.Seconds()) } @@ -84,7 +84,7 @@ func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { m.expCollectDuration.With(prometheus.Labels{ gcShardID: shardID, - gcSuccess: fmt.Sprintf("%v", success), + gcSuccess: strconv.FormatBool(success), gcObjectType: objectType, }).Add(d.Seconds()) } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 7f6f2d7d5..2440adcfb 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -46,7 +47,7 @@ func newMorphCacheMetrics() *morphCacheMetrics { func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { m.nnsContractHashDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } @@ -54,7 +55,7 @@ func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Dura func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { m.groupKeyDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } @@ -62,7 +63,7 @@ func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { m.txHeightDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 48a450c76..ae24c41b2 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -1,7 +1,7 @@ package metrics import ( - "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -50,18 +50,18 @@ func newTreeServiceMetrics() *treeServiceMetrics { func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { m.replicateTaskDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { m.replicateWaitDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { m.syncOpDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 67b9d8caf..b27e684d5 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "strconv" "sync" "time" @@ -148,7 +149,7 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { m.flushCounter.With(prometheus.Labels{ wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), + wcSuccess: strconv.FormatBool(success), wcStorage: storageType, }).Inc() } @@ -164,7 +165,7 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b m.With( prometheus.Labels{ wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), + wcSuccess: strconv.FormatBool(success), wcStorage: storageType, }, ).Observe(d.Seconds()) -- 2.45.2 From 71bbeddb64793cc5e7be7d4d144293dc1d38efcc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:03:37 +0300 Subject: [PATCH 112/233] [#424] metrics: Drop unused arg Signed-off-by: Dmitrii Stepanov --- pkg/metrics/morphcache.go | 5 ++--- pkg/morph/client/notary.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 2440adcfb..000ec0d80 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -6,7 +6,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" - "github.com/nspcc-dev/neo-go/pkg/util" "github.com/prometheus/client_golang/prometheus" ) @@ -20,7 +19,7 @@ type MorphCacheMetrics interface { AddGroupKeyDuration(success bool, d time.Duration) - AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) + AddTxHeightDuration(success bool, d time.Duration) } type morphCacheMetrics struct { @@ -60,7 +59,7 @@ func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { ).Observe(float64(d)) } -func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { +func (m *morphCacheMetrics) AddTxHeightDuration(success bool, d time.Duration) { m.txHeightDuration.With( prometheus.Labels{ mcSuccess: strconv.FormatBool(success), diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 680ba6a68..e567b6df0 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -800,7 +800,7 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddTxHeightDuration(h, success, time.Since(startedAt)) + c.cache.metrics.AddTxHeightDuration(success, time.Since(startedAt)) }() if rh, ok := c.cache.txHeights.Get(h); ok { -- 2.45.2 From 07f155ac77e3af09d849f271caf123eb0e71826d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:05:45 +0300 Subject: [PATCH 113/233] [#424] metrics: Use labels for writecache methods and operations Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 10 +- pkg/metrics/types.go | 17 +++ pkg/metrics/writecache.go | 110 ++++++------------ 3 files changed, 60 insertions(+), 77 deletions(-) create mode 100644 pkg/metrics/types.go diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index cd8278272..b947f12f4 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -63,18 +63,18 @@ type writeCacheMetrics struct { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddGetDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Get", success, d, st.String()) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddDeleteDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String()) if success { m.metrics.DecActualCount(m.shardID, st.String()) } } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddPutDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String()) if success { m.metrics.IncActualCount(m.shardID, st.String()) } @@ -95,10 +95,10 @@ func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { } func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { - m.metrics.IncFlushCounter(m.shardID, success, st.String()) + m.metrics.IncOperationCounter(m.shardID, "Flush", metrics.NullBool{Bool: success, Valid: true}, st.String()) } func (m *writeCacheMetrics) Evict(st writecache.StorageType) { m.metrics.DecActualCount(m.shardID, st.String()) - m.metrics.IncEvictCounter(m.shardID, st.String()) + m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go new file mode 100644 index 000000000..6a76248bf --- /dev/null +++ b/pkg/metrics/types.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "strconv" +) + +type NullBool struct { + Bool bool + Valid bool // Valid is true if Bool is not NULL +} + +func (v NullBool) String() string { + if !v.Valid { + return "" + } + return strconv.FormatBool(v.Bool) +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index b27e684d5..33c58fc65 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,7 +2,6 @@ package metrics import ( "fmt" - "strconv" "sync" "time" @@ -16,6 +15,8 @@ const ( wcSuccess = "success" wcStorage = "storage" wcMode = "mode" + wcMethod = "method" + wcOperation = "operation" ) type shardIDMode struct { @@ -23,9 +24,7 @@ type shardIDMode struct { } type WriteCacheMetrics interface { - AddGetDuration(shardID string, success bool, d time.Duration, storageType string) - AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) - AddPutDuration(shardID string, success bool, d time.Duration, storageType string) + AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) IncActualCount(shardID string, storageType string) DecActualCount(shardID string, storageType string) @@ -34,17 +33,12 @@ type WriteCacheMetrics interface { SetEstimateSize(shardID string, size uint64, storageType string) SetMode(shardID string, mode string) - IncFlushCounter(shardID string, success bool, storageType string) - IncEvictCounter(shardID string, storageType string) + IncOperationCounter(shardID string, operation string, success NullBool, storageType string) } type writeCacheMetrics struct { - getDuration *prometheus.HistogramVec - putDuration *prometheus.HistogramVec - deleteDuration *prometheus.HistogramVec - - flushCounter *prometheus.CounterVec - evictCounter *prometheus.CounterVec + methodDuration *prometheus.HistogramVec + operationCounter *prometheus.CounterVec actualCount *prometheus.GaugeVec @@ -57,29 +51,35 @@ type writeCacheMetrics struct { func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ - getDuration: newWCMethodDurationCounter("get"), - putDuration: newWCMethodDurationCounter("put"), - deleteDuration: newWCMethodDurationCounter("delete"), - flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), - evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), - actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), - estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), - modeValues: make(map[string]string), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "request_duration_seconds", + Help: "Writecache request process duration", + }, []string{wcShardID, wcSuccess, wcStorage, wcMethod}), + operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "operation_count", + Help: "The number of writecache operations processed", + }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), + actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), + modeMtx: sync.RWMutex{}, + modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), + modeValues: make(map[string]string), } } -func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.getDuration, shardID, success, d, storageType) -} - -func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.deleteDuration, shardID, success, d, storageType) -} - -func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.putDuration, shardID, success, d, storageType) +func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { + m.methodDuration.With( + prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + wcMethod: method, + }, + ).Observe(d.Seconds()) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { @@ -146,49 +146,15 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { m.modeMetrics[key] = metric } -func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { - m.flushCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcSuccess: strconv.FormatBool(success), - wcStorage: storageType, +func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { + m.operationCounter.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + wcOperation: operation, + wcSuccess: success.String(), }).Inc() } -func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { - m.evictCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, - }).Inc() -} - -func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { - m.With( - prometheus.Labels{ - wcShardID: shardID, - wcSuccess: strconv.FormatBool(success), - wcStorage: storageType, - }, - ).Observe(d.Seconds()) -} - -func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { - return metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_req_duration_seconds", method), - Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess, wcStorage}) -} - -func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { - return metrics.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_operation_count", operation), - Help: fmt.Sprintf("The number of %s operations processed", operation), - }, labels) -} - func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, -- 2.45.2 From 85deb12f4d078b5b3dcdd490a1b4c49ec6c9c0ce Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:14:28 +0300 Subject: [PATCH 114/233] [#424] writecache: Drop metrics when close Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 4 ++++ pkg/local_object_storage/writecache/metrics.go | 3 +++ pkg/local_object_storage/writecache/writecache.go | 1 + pkg/metrics/writecache.go | 10 ++++++++++ 4 files changed, 18 insertions(+) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index b947f12f4..2e518c6ff 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -102,3 +102,7 @@ func (m *writeCacheMetrics) Evict(st writecache.StorageType) { m.metrics.DecActualCount(m.shardID, st.String()) m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) } + +func (m *writeCacheMetrics) Close() { + m.metrics.Close(m.shardID) +} diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index e20c7e65e..957bf2770 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -28,6 +28,7 @@ type Metrics interface { SetEstimateSize(db, fstree uint64) SetMode(m mode.Mode) SetActualCounters(db, fstree uint64) + Close() } type metricsStub struct{} @@ -47,3 +48,5 @@ func (s *metricsStub) SetActualCounters(uint64, uint64) {} func (s *metricsStub) Flush(bool, StorageType) {} func (s *metricsStub) Evict(StorageType) {} + +func (s *metricsStub) Close() {} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 664beff80..6fa0e36de 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -170,5 +170,6 @@ func (c *cache) Close() error { c.db = nil } } + c.metrics.Close() return nil } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 33c58fc65..31636a295 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -34,6 +34,8 @@ type WriteCacheMetrics interface { SetMode(shardID string, mode string) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) + + Close(shardID string) } type writeCacheMetrics struct { @@ -155,6 +157,14 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string }).Inc() } +func (m *writeCacheMetrics) Close(shardID string) { + m.SetMode(shardID, "CLOSED") + m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) +} + func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, -- 2.45.2 From c8023a9c8d2c45ac03a53a62daaf218e44e399fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 16:15:59 +0300 Subject: [PATCH 115/233] [#424] morphcache: Use labels for method duration Signed-off-by: Dmitrii Stepanov --- pkg/metrics/morphcache.go | 58 +++++++++----------------------------- pkg/morph/client/nns.go | 4 +-- pkg/morph/client/notary.go | 2 +- 3 files changed, 16 insertions(+), 48 deletions(-) diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 000ec0d80..9cf2c8a0c 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -1,7 +1,6 @@ package metrics import ( - "fmt" "strconv" "time" @@ -12,66 +11,35 @@ import ( const ( mcSubsystem = "morphcache" mcSuccess = "success" + mcMethod = "method" ) type MorphCacheMetrics interface { - AddNNSContractHashDuration(success bool, d time.Duration) - - AddGroupKeyDuration(success bool, d time.Duration) - - AddTxHeightDuration(success bool, d time.Duration) + AddMethodDuration(method string, success bool, d time.Duration) } type morphCacheMetrics struct { - // Duration of processing get nns contract hash request - nnsContractHashDuration *prometheus.HistogramVec - - // Duration of processing get group key request - groupKeyDuration *prometheus.HistogramVec - - // Duration of processing get tx height request - txHeightDuration *prometheus.HistogramVec + methodDuration *prometheus.HistogramVec } var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) func newMorphCacheMetrics() *morphCacheMetrics { return &morphCacheMetrics{ - nnsContractHashDuration: newMCMethodDurationCounter("nns_contract_hash"), - groupKeyDuration: newMCMethodDurationCounter("group_key"), - txHeightDuration: newMCMethodDurationCounter("tx_height"), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: mcSubsystem, + Name: "request_duration_seconds", + Help: "Morph cache request process duration", + }, []string{mcSuccess, mcMethod}), } } -func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { - m.nnsContractHashDuration.With( +func (m *morphCacheMetrics) AddMethodDuration(method string, success bool, d time.Duration) { + m.methodDuration.With( prometheus.Labels{ mcSuccess: strconv.FormatBool(success), + mcMethod: method, }, - ).Observe(float64(d)) -} - -func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { - m.groupKeyDuration.With( - prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - }, - ).Observe(float64(d)) -} - -func (m *morphCacheMetrics) AddTxHeightDuration(success bool, d time.Duration) { - m.txHeightDuration.With( - prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - }, - ).Observe(float64(d)) -} - -func newMCMethodDurationCounter(method string) *prometheus.HistogramVec { - return metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: mcSubsystem, - Name: fmt.Sprintf("%s_req_duration_seconds", method), - Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{mcSuccess}) + ).Observe(d.Seconds()) } diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index d8d798b07..53dbe180e 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -86,7 +86,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { startedAt := time.Now() defer func() { - c.cache.metrics.AddNNSContractHashDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("NNSContractHash", success, time.Since(startedAt)) }() nnsHash := c.cache.nns() @@ -233,7 +233,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddGroupKeyDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("GroupKey", success, time.Since(startedAt)) }() if gKey := c.cache.groupKey(); gKey != nil { diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index e567b6df0..17644361a 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -800,7 +800,7 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddTxHeightDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("TxHeight", success, time.Since(startedAt)) }() if rh, ok := c.cache.txHeights.Get(h); ok { -- 2.45.2 From 1b364d8cf4db87b07909331584c86ee0347e1bdf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 19:48:15 +0300 Subject: [PATCH 116/233] [#424] metrics: Refactor engine metrics Use histogram vector to measure request duration. Fix naming like in Prometheus best practice. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/container.go | 4 +- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 2 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/metrics.go | 18 +-- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/engine/select.go | 4 +- pkg/local_object_storage/engine/shards.go | 6 +- .../shard/metrics_test.go | 2 +- pkg/local_object_storage/shard/shard.go | 8 +- pkg/metrics/engine.go | 114 ++++-------------- 13 files changed, 46 insertions(+), 122 deletions(-) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 034837110..061e2fea0 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -67,7 +67,7 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { if e.metrics != nil { - defer elapsed(e.metrics.AddEstimateContainerSizeDuration)() + defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() } e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -115,7 +115,7 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) { func (e *StorageEngine) listContainers() (ListContainersRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListContainersDuration)() + defer elapsed("ListContainers", e.metrics.AddMethodDuration)() } uniqueIDs := make(map[string]cid.ID) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 2125fad72..4d6d838bc 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -67,7 +67,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddDeleteDuration)() + defer elapsed("Delete", e.metrics.AddMethodDuration)() } var locked struct { diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index d376198ba..bd094770a 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -64,7 +64,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddGetDuration)() + defer elapsed("Get", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 5da97bab5..ca51015db 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -68,7 +68,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddHeadDuration)() + defer elapsed("Head", e.metrics.AddMethodDuration)() } var ( diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 57ac52751..0b9ae602b 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -78,7 +78,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddInhumeDuration)() + defer elapsed("Inhume", e.metrics.AddMethodDuration)() } var shPrm shard.InhumePrm diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index f9e9191cd..dd1240ea7 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -7,17 +7,7 @@ import ( ) type MetricRegister interface { - AddListContainersDuration(d time.Duration) - AddEstimateContainerSizeDuration(d time.Duration) - AddDeleteDuration(d time.Duration) - AddExistsDuration(d time.Duration) - AddGetDuration(d time.Duration) - AddHeadDuration(d time.Duration) - AddInhumeDuration(d time.Duration) - AddPutDuration(d time.Duration) - AddRangeDuration(d time.Duration) - AddSearchDuration(d time.Duration) - AddListObjectsDuration(d time.Duration) + AddMethodDuration(method string, d time.Duration) SetObjectCounter(shardID, objectType string, v uint64) AddToObjectCounter(shardID, objectType string, delta int) @@ -28,17 +18,17 @@ type MetricRegister interface { AddToPayloadCounter(shardID string, size int64) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) - DeleteErrorCounter(shardID string) + DeleteShardMetrics(shardID string) WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics } -func elapsed(addFunc func(d time.Duration)) func() { +func elapsed(method string, addFunc func(method string, d time.Duration)) func() { t := time.Now() return func() { - addFunc(time.Since(t)) + addFunc(method, time.Since(t)) } } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 4ac7f90f9..98c4504e6 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -57,7 +57,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { if e.metrics != nil { - defer elapsed(e.metrics.AddPutDuration)() + defer elapsed("Put", e.metrics.AddMethodDuration)() } addr := object.AddressOf(prm.obj) diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 29ec8b2bc..328df4587 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -80,7 +80,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddRangeDuration)() + defer elapsed("GetRange", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 48d2be674..9f651845f 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -60,7 +60,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddSearchDuration)() + defer elapsed("Search", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0) @@ -109,7 +109,7 @@ func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) { func (e *StorageEngine) list(limit uint64) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListObjectsDuration)() + defer elapsed("ListObjects", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0, limit) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index fd9605e9b..5f4f13635 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -70,8 +70,8 @@ func (m *metricsWithID) ClearErrorCounter() { m.mw.ClearErrorCounter(m.id) } -func (m *metricsWithID) DeleteErrorCounter() { - m.mw.DeleteErrorCounter(m.id) +func (m *metricsWithID) DeleteShardMetrics() { + m.mw.DeleteShardMetrics(m.id) } // AddShard adds a new shard to the storage engine. @@ -186,7 +186,7 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } - sh.DeleteErrorCounter() + sh.DeleteShardMetrics() ss = append(ss, sh) delete(e.shards, id) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index f1581b6d4..f9e1bc760 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -77,7 +77,7 @@ func (m *metricsStore) ClearErrorCounter() { m.errCounter = 0 } -func (m *metricsStore) DeleteErrorCounter() { +func (m *metricsStore) DeleteShardMetrics() { m.errCounter = 0 } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index b740fc572..8c4db87b5 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -79,8 +79,8 @@ type MetricsWriter interface { IncErrorCounter() // ClearErrorCounter clear error counter. ClearErrorCounter() - // DeleteErrorCounter delete error counter. - DeleteErrorCounter() + // DeleteShardMetrics deletes shard metrics from registry. + DeleteShardMetrics() } type cfg struct { @@ -447,8 +447,8 @@ func (s *Shard) ClearErrorCounter() { } } -func (s *Shard) DeleteErrorCounter() { +func (s *Shard) DeleteShardMetrics() { if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.DeleteErrorCounter() + s.cfg.metricsWriter.DeleteShardMetrics() } } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 7992da9f8..4b32344e4 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -1,8 +1,6 @@ package metrics import ( - "fmt" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -11,60 +9,33 @@ import ( type ( engineMetrics struct { - listContainersDuration prometheus.Counter - estimateContainerSizeDuration prometheus.Counter - deleteDuration prometheus.Counter - existsDuration prometheus.Counter - getDuration prometheus.Counter - headDuration prometheus.Counter - inhumeDuration prometheus.Counter - putDuration prometheus.Counter - rangeDuration prometheus.Counter - searchDuration prometheus.Counter - listObjectsDuration prometheus.Counter - containerSize *prometheus.GaugeVec - payloadSize *prometheus.GaugeVec - errorCounter *prometheus.GaugeVec + methodDuration *prometheus.HistogramVec + + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec } ) -const engineSubsystem = "engine" +const ( + engineSubsystem = "engine" + engineMethod = "method" +) func newEngineMetrics() engineMetrics { return engineMetrics{ - listContainersDuration: newEngineMethodDurationCounter("list_containers_"), - estimateContainerSizeDuration: newEngineCounter("estimate_container_size_duration", "Accumulated duration of engine container size estimate operations"), - deleteDuration: newEngineMethodDurationCounter("delete"), - existsDuration: newEngineMethodDurationCounter("exists"), - getDuration: newEngineMethodDurationCounter("get"), - headDuration: newEngineMethodDurationCounter("head"), - inhumeDuration: newEngineMethodDurationCounter("inhume"), - putDuration: newEngineMethodDurationCounter("put"), - rangeDuration: newEngineMethodDurationCounter("range"), - searchDuration: newEngineMethodDurationCounter("search"), - listObjectsDuration: newEngineMethodDurationCounter("list_objects"), - containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: engineSubsystem, + Name: "request_duration_seconds", + Help: "Duration of Engine requests", + }, []string{engineMethod}), } } -func newEngineCounter(name, help string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: name, - Help: help, - }) -} - -func newEngineMethodDurationCounter(method string) prometheus.Counter { - return newEngineCounter( - fmt.Sprintf("%s_duration", method), - fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), - ) -} - func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, @@ -74,48 +45,10 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV }, labels) } -func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) -} - -func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) -} - -func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.Add(float64(d)) -} - -func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.Add(float64(d)) -} - -func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.Add(float64(d)) -} - -func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.Add(float64(d)) -} - -func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) -} - -func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) +func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { + m.methodDuration.With(prometheus.Labels{ + engineMethod: method, + }).Observe(d.Seconds()) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { @@ -134,6 +67,7 @@ func (m engineMetrics) ClearErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) } -func (m engineMetrics) DeleteErrorCounter(shardID string) { +func (m engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) } -- 2.45.2 From c348ae35b0b7a856219e9ab98f2fc5e1ca31db72 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 10:05:51 +0300 Subject: [PATCH 117/233] [#424] metrics: Drop embedded metrics It was not obvious where metrics are used. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/control.go | 2 +- cmd/frostfs-node/object.go | 4 +- pkg/metrics/engine.go | 91 +++++++++++--- pkg/metrics/innerring.go | 8 +- pkg/metrics/node.go | 57 ++++----- pkg/metrics/object.go | 218 +++++---------------------------- pkg/metrics/replicator.go | 21 +++- pkg/metrics/state.go | 10 +- pkg/services/object/metrics.go | 45 ++----- 10 files changed, 173 insertions(+), 285 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 758803291..c88b7224e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -679,7 +679,7 @@ func (c *cfg) engineOpts() []engine.Option { ) if c.metricsCollector != nil { - opts = append(opts, engine.WithMetrics(c.metricsCollector)) + opts = append(opts, engine.WithMetrics(c.metricsCollector.Engine())) } return opts diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index f4b068419..3ed6bc54a 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -80,7 +80,7 @@ func (c *cfg) setHealthStatus(st control.HealthStatus) { c.healthStatus.Store(int32(st)) if c.metricsCollector != nil { - c.metricsCollector.SetHealth(int32(st)) + c.metricsCollector.State().SetHealth(int32(st)) } } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 7a409da03..0476dbcac 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -195,7 +195,7 @@ func initObjectService(c *cfg) { ) c.shared.metricsSvc = objectService.NewMetricCollector( - signSvc, c.metricsCollector, metricsconfig.Enabled(c.appCfg)) + signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) server := objectTransportGRPC.New(c.shared.metricsSvc) for _, srv := range c.cfgGRPC.servers { @@ -272,7 +272,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), - replicator.WithMetrics(c.metricsCollector), + replicator.WithMetrics(c.metricsCollector.Replicator()), ) } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4b32344e4..19e3591a6 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -7,23 +7,40 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -type ( - engineMetrics struct { - methodDuration *prometheus.HistogramVec +type EngineMetrics interface { + AddMethodDuration(method string, d time.Duration) + AddToContainerSize(cnrID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteShardMetrics(shardID string) + AddToObjectCounter(shardID, objectType string, delta int) + SetObjectCounter(shardID, objectType string, v uint64) + AddToPayloadCounter(shardID string, size int64) + SetReadonly(shardID string, readonly bool) - containerSize *prometheus.GaugeVec - payloadSize *prometheus.GaugeVec - errorCounter *prometheus.GaugeVec - } -) + WriteCache() WriteCacheMetrics + GC() GCMetrics +} + +type engineMetrics struct { + methodDuration *prometheus.HistogramVec + objectCounter *prometheus.GaugeVec + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec + shardsReadonly *prometheus.GaugeVec + + gc *gcMetrics + writeCache *writeCacheMetrics +} const ( engineSubsystem = "engine" engineMethod = "method" ) -func newEngineMetrics() engineMetrics { - return engineMetrics{ +func newEngineMetrics() *engineMetrics { + return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), @@ -33,6 +50,10 @@ func newEngineMetrics() engineMetrics { Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), + objectCounter: newEngineGaugeVector("object_counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), } } @@ -51,23 +72,63 @@ func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { }).Observe(d.Seconds()) } -func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { +func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } -func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { +func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } -func (m engineMetrics) IncErrorCounter(shardID string) { +func (m *engineMetrics) IncErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() } -func (m engineMetrics) ClearErrorCounter(shardID string) { +func (m *engineMetrics) ClearErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) } -func (m engineMetrics) DeleteShardMetrics(shardID string) { +func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) + m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID}) +} + +func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + counterTypeLabelKey: objectType, + }, + ).Add(float64(delta)) +} + +func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + counterTypeLabelKey: objectType, + }, + ).Set(float64(v)) +} + +func (m *engineMetrics) SetReadonly(shardID string, readonly bool) { + var flag float64 + if readonly { + flag = 1 + } + m.shardsReadonly.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + }, + ).Set(flag) +} + +func (m *engineMetrics) WriteCache() WriteCacheMetrics { + return m.writeCache +} + +func (m *engineMetrics) GC() GCMetrics { + return m.gc } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index d756a539f..b3cb3e285 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -54,22 +54,22 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { } // SetEpoch updates epoch metrics. -func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { +func (m *InnerRingServiceMetrics) SetEpoch(epoch uint64) { m.epoch.Set(float64(epoch)) } // SetHealth updates health metrics. -func (m InnerRingServiceMetrics) SetHealth(s int32) { +func (m *InnerRingServiceMetrics) SetHealth(s int32) { m.health.Set(float64(s)) } -func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { +func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { m.eventDuration.With(prometheus.Labels{ innerRingLabelType: typ, innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } -func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { +func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCacheMetrics } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 16a4c2b5b..232c178d8 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,15 +8,12 @@ import ( const namespace = "frostfs_node" type NodeMetrics struct { - objectServiceMetrics - engineMetrics - stateMetrics - replicatorMetrics - - writeCacheMetrics *writeCacheMetrics - treeService *treeServiceMetrics - epoch prometheus.Gauge - gc *gcMetrics + engine *engineMetrics + state *stateMetrics + replicator *replicatorMetrics + objectService *objectServiceMetrics + treeService *treeServiceMetrics + epoch prometheus.Gauge } func NewNodeMetrics() *NodeMetrics { @@ -37,19 +34,13 @@ func NewNodeMetrics() *NodeMetrics { Help: "Current epoch as seen by inner-ring node.", }) - writeCacheMetrics := newWriteCacheMetrics() - - gc := newGCMetrics() - return &NodeMetrics{ - objectServiceMetrics: objectService, - engineMetrics: engine, - stateMetrics: state, - replicatorMetrics: replicator, - treeService: treeService, - epoch: epoch, - writeCacheMetrics: writeCacheMetrics, - gc: gc, + objectService: objectService, + engine: engine, + state: state, + replicator: replicator, + treeService: treeService, + epoch: epoch, } } @@ -58,18 +49,22 @@ func (m *NodeMetrics) SetEpoch(epoch uint64) { m.epoch.Set(float64(epoch)) } -// WriteCache returns WriteCache metrics. -func (m *NodeMetrics) WriteCache() WriteCacheMetrics { - if m == nil { - return nil - } - return m.writeCacheMetrics -} - func (m *NodeMetrics) TreeService() TreeMetricsRegister { return m.treeService } -func (m *NodeMetrics) GC() GCMetrics { - return m.gc +func (m *NodeMetrics) Replicator() ReplicatorMetrics { + return m.replicator +} + +func (m *NodeMetrics) ObjectService() ObjectServiceMetrics { + return m.objectService +} + +func (m *NodeMetrics) Engine() EngineMetrics { + return m.engine +} + +func (m *NodeMetrics) State() StateMetrics { + return m.state } diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 87916414d..6e1b3a5c4 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -1,8 +1,7 @@ package metrics import ( - "fmt" - "strings" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -11,207 +10,50 @@ import ( const objectSubsystem = "object" -type ( - methodCount struct { - success prometheus.Counter - total prometheus.Counter - } +type ObjectServiceMetrics interface { + AddRequestDuration(method string, d time.Duration, success bool) + AddPayloadSize(method string, size int) +} - objectServiceMetrics struct { - getCounter methodCount - putCounter methodCount - headCounter methodCount - searchCounter methodCount - deleteCounter methodCount - rangeCounter methodCount - rangeHashCounter methodCount - - getDuration prometheus.Counter - putDuration prometheus.Counter - headDuration prometheus.Counter - searchDuration prometheus.Counter - deleteDuration prometheus.Counter - rangeDuration prometheus.Counter - rangeHashDuration prometheus.Counter - - putPayload prometheus.Counter - getPayload prometheus.Counter - - shardMetrics *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec - } -) +type objectServiceMetrics struct { + methodDuration *prometheus.HistogramVec + payloadCounter *prometheus.CounterVec +} const ( shardIDLabelKey = "shard" counterTypeLabelKey = "type" containerIDLabelKey = "cid" + methodLabelKey = "method" + successLabelKey = "success" ) -func newObjectMethodCallCounter(name string) methodCount { - return methodCount{ - success: metrics.NewCounter(prometheus.CounterOpts{ +func newObjectServiceMetrics() *objectServiceMetrics { + return &objectServiceMetrics{ + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count_success", name), - Help: fmt.Sprintf("The number of successful %s requests processed", name), - }), - total: metrics.NewCounter(prometheus.CounterOpts{ + Name: "request_duration_seconds", + Help: "Object Service request process duration", + }, []string{methodLabelKey, successLabelKey}), + payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count", name), - Help: fmt.Sprintf("Total number of %s requests processed", name), - }), + Name: "request_payload_bytes", + Help: "Object Service request payload", + }, []string{methodLabelKey}), } } -func (m methodCount) Inc(success bool) { - m.total.Inc() - if success { - m.success.Inc() - } +func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { + m.methodDuration.With(prometheus.Labels{ + methodLabelKey: method, + successLabelKey: strconv.FormatBool(success), + }).Observe(d.Seconds()) } -func newObjectServiceMetrics() objectServiceMetrics { - return objectServiceMetrics{ - getCounter: newObjectMethodCallCounter("get"), - putCounter: newObjectMethodCallCounter("put"), - headCounter: newObjectMethodCallCounter("head"), - searchCounter: newObjectMethodCallCounter("search"), - deleteCounter: newObjectMethodCallCounter("delete"), - rangeCounter: newObjectMethodCallCounter("range"), - rangeHashCounter: newObjectMethodCallCounter("range_hash"), - getDuration: newObjectMethodDurationCounter("get"), - putDuration: newObjectMethodDurationCounter("put"), - headDuration: newObjectMethodDurationCounter("head"), - searchDuration: newObjectMethodDurationCounter("search"), - deleteDuration: newObjectMethodDurationCounter("delete"), - rangeDuration: newObjectMethodDurationCounter("range"), - rangeHashDuration: newObjectMethodDurationCounter("range_hash"), - putPayload: newObjectMethodPayloadCounter("put"), - getPayload: newObjectMethodPayloadCounter("get"), - shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), - shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}), - } -} - -func newObjectMethodPayloadCounter(method string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_payload", method), - Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectMethodDurationCounter(method string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_duration", method), - Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return metrics.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: name, - Help: help, - }, labels) -} - -func (m objectServiceMetrics) IncGetReqCounter(success bool) { - m.getCounter.Inc(success) -} - -func (m objectServiceMetrics) IncPutReqCounter(success bool) { - m.putCounter.Inc(success) -} - -func (m objectServiceMetrics) IncHeadReqCounter(success bool) { - m.headCounter.Inc(success) -} - -func (m objectServiceMetrics) IncSearchReqCounter(success bool) { - m.searchCounter.Inc(success) -} - -func (m objectServiceMetrics) IncDeleteReqCounter(success bool) { - m.deleteCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeReqCounter(success bool) { - m.rangeCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { - m.rangeHashCounter.Inc(success) -} - -func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Add(float64(delta)) -} - -func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Set(float64(v)) -} - -func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { - var flag float64 - if readonly { - flag = 1 - } - m.shardsReadonly.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - }, - ).Set(flag) +func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { + m.payloadCounter.With(prometheus.Labels{ + methodLabelKey: method, + }).Add(float64(size)) } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index 0deafe916..dbcc67338 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -7,30 +7,39 @@ import ( const replicatorSubsystem = "replicator" +//TODO + +type ReplicatorMetrics interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} + type replicatorMetrics struct { inFlightRequests prometheus.Gauge processedObjects prometheus.Counter totalReplicatedPayloadSize prometheus.Counter } -func (m replicatorMetrics) IncInFlightRequest() { +func (m *replicatorMetrics) IncInFlightRequest() { m.inFlightRequests.Inc() } -func (m replicatorMetrics) DecInFlightRequest() { +func (m *replicatorMetrics) DecInFlightRequest() { m.inFlightRequests.Dec() } -func (m replicatorMetrics) IncProcessedObjects() { +func (m *replicatorMetrics) IncProcessedObjects() { m.processedObjects.Inc() } -func (m replicatorMetrics) AddPayloadSize(size int64) { +func (m *replicatorMetrics) AddPayloadSize(size int64) { m.totalReplicatedPayloadSize.Add(float64(size)) } -func newReplicatorMetrics() replicatorMetrics { - return replicatorMetrics{ +func newReplicatorMetrics() *replicatorMetrics { + return &replicatorMetrics{ inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 893849911..76a8d8ac9 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -7,12 +7,16 @@ import ( const stateSubsystem = "state" +type StateMetrics interface { + SetHealth(s int32) +} + type stateMetrics struct { healthCheck prometheus.Gauge } -func newStateMetrics() stateMetrics { - return stateMetrics{ +func newStateMetrics() *stateMetrics { + return &stateMetrics{ healthCheck: metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, @@ -22,6 +26,6 @@ func newStateMetrics() stateMetrics { } } -func (m stateMetrics) SetHealth(s int32) { +func (m *stateMetrics) SetHealth(s int32) { m.healthCheck.Set(float64(s)) } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 3ea16dafd..487374940 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -28,24 +28,8 @@ type ( } MetricRegister interface { - IncGetReqCounter(success bool) - IncPutReqCounter(success bool) - IncHeadReqCounter(success bool) - IncSearchReqCounter(success bool) - IncDeleteReqCounter(success bool) - IncRangeReqCounter(success bool) - IncRangeHashReqCounter(success bool) - - AddGetReqDuration(time.Duration) - AddPutReqDuration(time.Duration) - AddHeadReqDuration(time.Duration) - AddSearchReqDuration(time.Duration) - AddDeleteReqDuration(time.Duration) - AddRangeReqDuration(time.Duration) - AddRangeHashReqDuration(time.Duration) - - AddPutPayload(int) - AddGetPayload(int) + AddRequestDuration(string, time.Duration, bool) + AddPayloadSize(string, int) } ) @@ -61,8 +45,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er if m.enabled { t := time.Now() defer func() { - m.metrics.IncGetReqCounter(err == nil) - m.metrics.AddGetReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Get", time.Since(t), err == nil) }() err = m.next.Get(req, &getStreamMetric{ ServerStream: stream, @@ -99,8 +82,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) res, err := m.next.Head(ctx, request) - m.metrics.IncHeadReqCounter(err == nil) - m.metrics.AddHeadReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Head", time.Since(t), err == nil) return res, err } @@ -113,8 +95,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) err := m.next.Search(req, stream) - m.metrics.IncSearchReqCounter(err == nil) - m.metrics.AddSearchReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Search", time.Since(t), err == nil) return err } @@ -127,8 +108,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque res, err := m.next.Delete(ctx, request) - m.metrics.IncDeleteReqCounter(err == nil) - m.metrics.AddDeleteReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil) return res, err } return m.next.Delete(ctx, request) @@ -140,8 +120,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR err := m.next.GetRange(req, stream) - m.metrics.IncRangeReqCounter(err == nil) - m.metrics.AddRangeReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil) return err } @@ -154,8 +133,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa res, err := m.next.GetRangeHash(ctx, request) - m.metrics.IncRangeHashReqCounter(err == nil) - m.metrics.AddRangeHashReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil) return res, err } @@ -173,7 +151,7 @@ func (m *MetricCollector) Disable() { func (s getStreamMetric) Send(resp *object.GetResponse) error { chunk, ok := resp.GetBody().GetObjectPart().(*object.GetObjectPartChunk) if ok { - s.metrics.AddGetPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Get", len(chunk.GetChunk())) } return s.stream.Send(resp) @@ -182,7 +160,7 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error { func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error { chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk) if ok { - s.metrics.AddPutPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Put", len(chunk.GetChunk())) } return s.stream.Send(ctx, req) @@ -191,8 +169,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { res, err := s.stream.CloseAndRecv(ctx) - s.metrics.IncPutReqCounter(err == nil) - s.metrics.AddPutReqDuration(time.Since(s.start)) + s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil) return res, err } -- 2.45.2 From 847732605c025d6796dcb7923cccfe987cda6bbd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 10:16:19 +0300 Subject: [PATCH 118/233] [#424] metrics: Rename counter to total Suffix total should be used for a unit-less accumulating count. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/engine.go | 4 ++-- pkg/metrics/gc.go | 4 ++-- pkg/metrics/morph.go | 4 ++-- pkg/metrics/replicator.go | 6 +++--- pkg/metrics/writecache.go | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 19e3591a6..d3f3238ac 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -43,14 +43,14 @@ func newEngineMetrics() *engineMetrics { return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabelKey}), methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("object_counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index 9c00d8722..c0319562c 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -43,7 +43,7 @@ func newGCMetrics() *gcMetrics { deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, - Name: "deleted_objects_count", + Name: "deleted_objects_total", Help: "Total count of objects GC deleted or failed to delete from disk", }, []string{gcShardID, gcStatus}), expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ @@ -55,7 +55,7 @@ func newGCMetrics() *gcMetrics { inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, - Name: "marked_for_removal_objects_count", + Name: "marked_for_removal_objects_total", Help: "Total count of expired objects GC marked to remove", }, []string{gcShardID, gcObjectType}), } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go index cc851d3db..cd5deb5e6 100644 --- a/pkg/metrics/morph.go +++ b/pkg/metrics/morph.go @@ -30,7 +30,7 @@ func NewMorphClientMetrics() morphmetrics.Register { switchCount: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: morphSubsystem, - Name: "switch_count", + Name: "switches_total", Help: "Number of endpoint switches", }), lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ @@ -42,7 +42,7 @@ func NewMorphClientMetrics() morphmetrics.Register { notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: morphSubsystem, - Name: "notification_count", + Name: "notifications_total", Help: "Number of notifications received by notification type", }, []string{morphNotificationTypeLabel}), invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index dbcc67338..f16c296c6 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -40,9 +40,9 @@ func (m *replicatorMetrics) AddPayloadSize(size int64) { func newReplicatorMetrics() *replicatorMetrics { return &replicatorMetrics{ - inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), - processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), - totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), + inFlightRequests: newReplicatorGauge("in_flight_requests_total", "Number of in-flight requests"), + processedObjects: newReplicatorCounter("processed_objects_total", "Number of objects processed since the node startup"), + totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size_bytes", "Total size of payloads replicated"), } } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 31636a295..8498e3930 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -62,10 +62,10 @@ func newWriteCacheMetrics() *writeCacheMetrics { operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: wcSubsystem, - Name: "operation_count", + Name: "operations_total", Help: "The number of writecache operations processed", }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), - actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), modeMtx: sync.RWMutex{}, modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), -- 2.45.2 From 4449006862cafebaafc67424a2fc777e3f85006b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 11:00:44 +0300 Subject: [PATCH 119/233] [#424] metrics: Use mode value as metric value for shard Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 3 +- pkg/local_object_storage/engine/shards.go | 6 +-- .../shard/metrics_test.go | 10 ++-- pkg/local_object_storage/shard/mode.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +- pkg/metrics/engine.go | 27 ++++------ pkg/metrics/mode.go | 39 +++++++++++++++ pkg/metrics/writecache.go | 49 ++----------------- 8 files changed, 66 insertions(+), 74 deletions(-) create mode 100644 pkg/metrics/mode.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index dd1240ea7..fcac2dc60 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -3,6 +3,7 @@ package engine import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -12,7 +13,7 @@ type MetricRegister interface { SetObjectCounter(shardID, objectType string, v uint64) AddToObjectCounter(shardID, objectType string, delta int) - SetReadonly(shardID string, readonly bool) + SetMode(shardID string, mode mode.Mode) AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 5f4f13635..cfa64929b 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -50,8 +50,8 @@ func (m *metricsWithID) DecObjectCounter(objectType string) { m.mw.AddToObjectCounter(m.id, objectType, -1) } -func (m *metricsWithID) SetReadonly(readonly bool) { - m.mw.SetReadonly(m.id, readonly) +func (m *metricsWithID) SetMode(mode mode.Mode) { + m.mw.SetMode(m.id, mode) } func (m *metricsWithID) AddToContainerSize(cnr string, size int64) { @@ -90,7 +90,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) { } if e.cfg.metrics != nil { - e.cfg.metrics.SetReadonly(sh.ID().String(), sh.GetMode() != mode.ReadWrite) + e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) } return sh.ID(), nil diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index f9e1bc760..d76c240b7 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -22,7 +22,7 @@ type metricsStore struct { objCounters map[string]uint64 cnrSize map[string]int64 pldSize int64 - readOnly bool + mode mode.Mode errCounter int64 } @@ -57,8 +57,8 @@ func (m metricsStore) DecObjectCounter(objectType string) { m.AddToObjectCounter(objectType, -1) } -func (m *metricsStore) SetReadonly(r bool) { - m.readOnly = r +func (m *metricsStore) SetMode(mode mode.Mode) { + m.mode = mode } func (m metricsStore) AddToContainerSize(cnr string, size int64) { @@ -91,9 +91,9 @@ func TestCounters(t *testing.T) { sh, mm := shardWithMetrics(t, dir) sh.SetMode(mode.ReadOnly) - require.True(t, mm.readOnly) + require.Equal(t, mode.ReadOnly, mm.mode) sh.SetMode(mode.ReadWrite) - require.False(t, mm.readOnly) + require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 oo := make([]*object.Object, objNumber) diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index efd41863b..a59f08704 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -64,7 +64,7 @@ func (s *Shard) setMode(m mode.Mode) error { s.info.Mode = m if s.metricsWriter != nil { - s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite) + s.metricsWriter.SetMode(s.info.Mode) } s.log.Info(logs.ShardShardModeSetSuccessfully, diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 8c4db87b5..1d2cab9f2 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -73,8 +73,8 @@ type MetricsWriter interface { // SetShardID must set (update) the shard identifier that will be used in // metrics. SetShardID(id string) - // SetReadonly must set shard readonly state. - SetReadonly(readonly bool) + // SetReadonly must set shard mode. + SetMode(mode mode.Mode) // IncErrorCounter increment error counter. IncErrorCounter() // ClearErrorCounter clear error counter. diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index d3f3238ac..7eac85861 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -3,6 +3,7 @@ package metrics import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -16,7 +17,7 @@ type EngineMetrics interface { AddToObjectCounter(shardID, objectType string, delta int) SetObjectCounter(shardID, objectType string, v uint64) AddToPayloadCounter(shardID string, size int64) - SetReadonly(shardID string, readonly bool) + SetMode(shardID string, mode mode.Mode) WriteCache() WriteCacheMetrics GC() GCMetrics @@ -28,7 +29,7 @@ type engineMetrics struct { containerSize *prometheus.GaugeVec payloadSize *prometheus.GaugeVec errorCounter *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec + mode *shardIDModeValue gc *gcMetrics writeCache *writeCacheMetrics @@ -50,10 +51,10 @@ func newEngineMetrics() *engineMetrics { Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), - shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), - gc: newGCMetrics(), - writeCache: newWriteCacheMetrics(), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), + mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"), } } @@ -92,7 +93,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) - m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.mode.Delete(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { @@ -113,16 +114,8 @@ func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { ).Set(float64(v)) } -func (m *engineMetrics) SetReadonly(shardID string, readonly bool) { - var flag float64 - if readonly { - flag = 1 - } - m.shardsReadonly.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - }, - ).Set(flag) +func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) } func (m *engineMetrics) WriteCache() WriteCacheMetrics { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go new file mode 100644 index 000000000..7ae3fd5f9 --- /dev/null +++ b/pkg/metrics/mode.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type shardIDModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDMode(subsystem, name, help string) *shardIDModeValue { + return &shardIDModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{wcShardID, wcMode}), + } +} + +func (m *shardIDModeValue) SetMode(shardID string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + wcShardID: shardID, + }) + + m.modeValue.With(prometheus.Labels{ + wcShardID: shardID, + wcMode: mode, + }).Set(1) +} + +func (m *shardIDModeValue) Delete(shardID string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + wcShardID: shardID, + }) +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 8498e3930..373772a62 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,7 +2,6 @@ package metrics import ( "fmt" - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -19,10 +18,6 @@ const ( wcOperation = "operation" ) -type shardIDMode struct { - shardID, mode string -} - type WriteCacheMetrics interface { AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) @@ -46,9 +41,7 @@ type writeCacheMetrics struct { estimatedSize *prometheus.GaugeVec - modeMetrics map[shardIDMode]prometheus.GaugeFunc - modeValues map[string]string - modeMtx sync.RWMutex + mode *shardIDModeValue } func newWriteCacheMetrics() *writeCacheMetrics { @@ -67,9 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), - modeValues: make(map[string]string), + mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"), } } @@ -113,39 +104,7 @@ func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storage } func (m *writeCacheMetrics) SetMode(shardID string, mode string) { - m.modeMtx.Lock() - defer m.modeMtx.Unlock() - - m.modeValues[shardID] = mode - key := shardIDMode{ - shardID: shardID, - mode: mode, - } - if _, found := m.modeMetrics[key]; found { - return - } - - metric := metrics.NewGaugeFunc( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: "writecache_mode", - Help: "Writecache mode value", - ConstLabels: prometheus.Labels{ - wcShardID: shardID, - wcMode: mode, - }, - }, func() float64 { - m.modeMtx.RLock() - defer m.modeMtx.RUnlock() - - value := m.modeValues[shardID] - if value == mode { - return 1 - } - return 0 - }) - m.modeMetrics[key] = metric + m.mode.SetMode(shardID, mode) } func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { @@ -158,7 +117,7 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string } func (m *writeCacheMetrics) Close(shardID string) { - m.SetMode(shardID, "CLOSED") + m.mode.Delete(shardID) m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) -- 2.45.2 From 26b305f82b830eebf6c727f013606b576299e45d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 12:12:47 +0300 Subject: [PATCH 120/233] [#424] morph: Fix cache metrics Use separate morph cache metrics for node and IR Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/morph.go | 3 +-- pkg/metrics/innerring.go | 9 +++++---- pkg/metrics/morphcache.go | 8 ++++++-- pkg/morph/client/constructor.go | 1 + pkg/morph/metrics/metrics.go | 4 ++++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index ae50b8a8a..63d1605ef 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -30,8 +30,6 @@ const ( ) func initMorphComponents(ctx context.Context, c *cfg) { - var err error - addresses := morphconfig.RPCEndpoint(c.appCfg) // Morph client stable-sorts endpoints by priority. Shuffle here to randomize @@ -50,6 +48,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.internalErr <- errors.New("morph connection has been lost") }), client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), + client.WithMorphCacheMetrics(metrics.NewNodeMorphCacheMetrics()), ) if err != nil { c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index b3cb3e285..0aee068c5 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -12,6 +12,7 @@ const ( innerRingSubsystem = "ir" innerRingLabelSuccess = "success" innerRingLabelType = "type" + innerRingNamespace = "frostfs_ir" ) // InnerRingServiceMetrics contains metrics collected by inner ring. @@ -26,19 +27,19 @@ type InnerRingServiceMetrics struct { func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( epoch = metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) health = metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "health", Help: "Current inner-ring node state.", }) eventDuration = metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "event_duration_seconds", Help: "Duration of processing of inner-ring events", @@ -49,7 +50,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { epoch: epoch, health: health, eventDuration: eventDuration, - morphCacheMetrics: newMorphCacheMetrics(), + morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), } } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 9cf2c8a0c..3f215b5bf 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -24,10 +24,14 @@ type morphCacheMetrics struct { var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) -func newMorphCacheMetrics() *morphCacheMetrics { +func NewNodeMorphCacheMetrics() MorphCacheMetrics { + return newMorphCacheMetrics(namespace) +} + +func newMorphCacheMetrics(ns string) *morphCacheMetrics { return &morphCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, + Namespace: ns, Subsystem: mcSubsystem, Name: "request_duration_seconds", Help: "Morph cache request process duration", diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 6cc66c5cc..e7e1bbca9 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -68,6 +68,7 @@ func defaultConfig() *cfg { signer: &transaction.Signer{ Scopes: transaction.Global, }, + morphCacheMetrics: &morphmetrics.NoopMorphCacheMetrics{}, } } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go index 9e41a0b86..5d74b054d 100644 --- a/pkg/morph/metrics/metrics.go +++ b/pkg/morph/metrics/metrics.go @@ -15,3 +15,7 @@ func (NoopRegister) IncSwitchCount() { func (NoopRegister) SetLastBlock(uint32) {} func (NoopRegister) IncNotificationCount(string) {} func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} + +type NoopMorphCacheMetrics struct{} + +func (m *NoopMorphCacheMetrics) AddMethodDuration(string, bool, time.Duration) {} -- 2.45.2 From 69b788a90b437b04da17996166361f016e8c0e25 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 18:39:49 +0300 Subject: [PATCH 121/233] [#424] metrics: Rename mode to mode_info To follow best practice. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/engine.go | 2 +- pkg/metrics/writecache.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 7eac85861..395ab3356 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -54,7 +54,7 @@ func newEngineMetrics() *engineMetrics { objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), - mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"), + mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), } } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 373772a62..a9c1b33a7 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -60,7 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"), + mode: newShardIDMode(wcSubsystem, "mode_info", "Writecache mode value"), } } -- 2.45.2 From 20b84f183a2936fe3b4522f13ee06468df63ef98 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 11:42:54 +0300 Subject: [PATCH 122/233] [#446] engine: Simplify logs for shard mode change Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 7c36811e7..9967b2e2b 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -83,32 +83,28 @@ func (e *StorageEngine) setModeLoop() { } func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { + sid := sh.ID() + log := e.log.With( + zap.Stringer("shard_id", sid), + zap.Uint32("error count", errCount)) + e.mtx.RLock() defer e.mtx.RUnlock() - sid := sh.ID() err := sh.SetMode(mode.DegradedReadOnly) if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), + log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) err = sh.SetMode(mode.ReadOnly) if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) } else { - e.log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } } else { - e.log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } } -- 2.45.2 From fe0178181171cff0bf00e7b24943d15dedacb77b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 13:19:36 +0300 Subject: [PATCH 123/233] [#446] los: Wrap SSD errors in a separate type Signed-off-by: Evgenii Stratonikov --- .../internal/metaerr/error.go | 33 +++++++++ .../internal/metaerr/error_test.go | 67 +++++++++++++++++++ pkg/local_object_storage/metabase/children.go | 3 +- .../metabase/containers.go | 5 +- pkg/local_object_storage/metabase/control.go | 15 +++-- pkg/local_object_storage/metabase/counter.go | 3 +- pkg/local_object_storage/metabase/delete.go | 3 +- pkg/local_object_storage/metabase/exists.go | 3 +- pkg/local_object_storage/metabase/expired.go | 3 +- pkg/local_object_storage/metabase/get.go | 3 +- .../metabase/graveyard.go | 9 +-- pkg/local_object_storage/metabase/inhume.go | 3 +- .../metabase/iterators.go | 5 +- pkg/local_object_storage/metabase/list.go | 3 +- pkg/local_object_storage/metabase/lock.go | 11 +-- pkg/local_object_storage/metabase/movable.go | 11 +-- pkg/local_object_storage/metabase/put.go | 3 +- pkg/local_object_storage/metabase/select.go | 5 +- pkg/local_object_storage/metabase/shard_id.go | 7 +- .../metabase/storage_id.go | 5 +- pkg/local_object_storage/pilorama/boltdb.go | 45 +++++++------ pkg/local_object_storage/writecache/delete.go | 3 +- pkg/local_object_storage/writecache/flush.go | 9 +-- pkg/local_object_storage/writecache/get.go | 8 ++- .../writecache/iterate.go | 5 +- pkg/local_object_storage/writecache/put.go | 3 +- .../writecache/writecache.go | 5 +- 27 files changed, 202 insertions(+), 76 deletions(-) create mode 100644 pkg/local_object_storage/internal/metaerr/error.go create mode 100644 pkg/local_object_storage/internal/metaerr/error_test.go diff --git a/pkg/local_object_storage/internal/metaerr/error.go b/pkg/local_object_storage/internal/metaerr/error.go new file mode 100644 index 000000000..41b8504bc --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error.go @@ -0,0 +1,33 @@ +package metaerr + +import "errors" + +// Error is a wrapper for SSD-related errors. +// In our model it unites metabase, pilorama and write-cache errors. +type Error struct { + err error +} + +// New returns simple error with a provided error message. +func New(msg string) Error { + return Error{err: errors.New(msg)} +} + +// Error implements the error interface. +func (e Error) Error() string { + return e.err.Error() +} + +// Wrap wraps arbitrary error. +// Returns nil if err == nil. +func Wrap(err error) error { + if err != nil { + return Error{err: err} + } + return nil +} + +// Unwrap returns underlying error. +func (e Error) Unwrap() error { + return e.err +} diff --git a/pkg/local_object_storage/internal/metaerr/error_test.go b/pkg/local_object_storage/internal/metaerr/error_test.go new file mode 100644 index 000000000..5a16aa501 --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error_test.go @@ -0,0 +1,67 @@ +package metaerr + +import ( + "errors" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestError(t *testing.T) { + t.Run("errors.Is", func(t *testing.T) { + e1 := errors.New("some error") + ee := Wrap(e1) + require.ErrorIs(t, ee, e1) + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + require.ErrorIs(t, ee, e1) + require.ErrorIs(t, ee, e2) + + require.Equal(t, errors.Unwrap(ee), e2) + }) + + t.Run("errors.As", func(t *testing.T) { + e1 := testError{42} + ee := Wrap(e1) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + { + var actual Error + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1, actual.err) + } + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + }) +} +func TestNilWrap(t *testing.T) { + require.NoError(t, Wrap(nil)) +} + +func TestErrorMessage(t *testing.T) { + msg := "sth to report" + err := New(msg) + require.Contains(t, err.Error(), msg) +} + +type testError struct { + data uint64 +} + +func (e testError) Error() string { + return strconv.FormatUint(e.data, 10) +} diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go index f0591b43c..500e83e7a 100644 --- a/pkg/local_object_storage/metabase/children.go +++ b/pkg/local_object_storage/metabase/children.go @@ -1,6 +1,7 @@ package meta import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -50,7 +51,7 @@ func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Addres }) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return result, nil diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index 3d69649a9..fe38e0b6d 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -3,6 +3,7 @@ package meta import ( "encoding/binary" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) @@ -21,7 +22,7 @@ func (db *DB) Containers() (list []cid.ID, err error) { return err }) - return list, err + return list, metaerr.Wrap(err) } func (db *DB) containers(tx *bbolt.Tx) ([]cid.ID, error) { @@ -55,7 +56,7 @@ func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { return err }) - return size, err + return size, metaerr.Wrap(err) } func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) (uint64, error) { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 4ae802aaa..370fddda1 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -6,6 +6,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -34,7 +35,7 @@ func (db *DB) Open(readOnly bool) error { } db.boltOptions.ReadOnly = readOnly - return db.openBolt() + return metaerr.Wrap(db.openBolt()) } func (db *DB) openBolt() error { @@ -79,7 +80,7 @@ func (db *DB) openBolt() error { // Does nothing if metabase has already been initialized and filled. To roll back the database to its initial state, // use Reset. func (db *DB) Init() error { - return db.init(false) + return metaerr.Wrap(db.init(false)) } // Reset resets metabase. Works similar to Init but cleans up all static buckets and @@ -92,7 +93,7 @@ func (db *DB) Reset() error { return ErrDegradedMode } - return db.init(true) + return metaerr.Wrap(db.init(true)) } func (db *DB) init(reset bool) error { @@ -167,15 +168,15 @@ func (db *DB) SyncCounters() error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { return syncCounter(tx, true) - }) + })) } // Close closes boltDB instance. func (db *DB) Close() error { if db.boltDB != nil { - return db.boltDB.Close() + return metaerr.Wrap(db.boltDB.Close()) } return nil } @@ -203,7 +204,7 @@ func (db *DB) Reload(opts ...Option) (bool, error) { db.mode = mode.Degraded db.info.Path = c.info.Path if err := db.openBolt(); err != nil { - return false, fmt.Errorf("%w: %v", ErrDegradedMode, err) + return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index a07328026..c0dc7886e 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -67,7 +68,7 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { return nil }) - return + return cc, metaerr.Wrap(err) } // updateCounter updates the object counter. Tx MUST be writable. diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 6ddf96593..7e8e0e5dd 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -108,7 +109,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { availableRemoved: availableRemoved, sizes: sizes, availableSizes: availableSizes, - }, err + }, metaerr.Wrap(err) } // deleteGroup deletes object from the metabase. Handles removal of the diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 74faf1634..f9c563bdd 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -5,6 +5,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -65,7 +66,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists bool, err error) { diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index d20fdbfa9..aac158ba4 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -7,6 +7,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -65,7 +66,7 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A }) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return result, nil } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 2f94c72d0..2ce82bcee 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -74,7 +75,7 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 393c9f4d0..e2530bd31 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -65,9 +66,9 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, gcHandler{p.h}, p.offset) - }) + })) } // TombstonedObject represents descriptor of the @@ -132,9 +133,9 @@ func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, graveyardHandler{p.h}, p.offset) - }) + })) } type kvHandler interface { diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index e4af1d1b9..7865a8771 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -138,7 +139,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err return db.inhumeTx(tx, currEpoch, prm, &res) }) - return + return res, metaerr.Wrap(err) } func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes) error { diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index ba22566a3..dfa048621 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -6,6 +6,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -51,9 +52,9 @@ func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateExpired(tx, epoch, h) - }) + })) } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 7fe1040bb..7c77cd703 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -2,6 +2,7 @@ package meta import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -75,7 +76,7 @@ func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { return err }) - return res, err + return res, metaerr.Wrap(err) } func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, count int, cursor *Cursor) ([]objectcore.AddressWithType, *Cursor, error) { diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 2b5fcd775..b84e0bf46 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -63,7 +64,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } key := make([]byte, cidSize) - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != object.TypeRegular { return logicerr.Wrap(apistatus.LockNonRegularObject{}) } @@ -109,7 +110,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } return nil - }) + })) } // FreeLockedBy unlocks all objects in DB which are locked by lockers. @@ -135,7 +136,7 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { return nil }); err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return unlockedObjects, nil } @@ -292,8 +293,8 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e return res, ErrDegradedMode } - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.locked = objectLocked(tx, prm.addr.Container(), prm.addr.Object()) return nil - }) + })) } diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index 033b9c718..763e49a5d 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -76,7 +77,7 @@ func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, e return toMoveIt.Put(key, zeroValue) }) - return + return res, metaerr.Wrap(err) } // DoNotMove removes `MoveIt` mark from the object. @@ -98,7 +99,7 @@ func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) { return toMoveIt.Delete(key) }) - return + return res, metaerr.Wrap(err) } // Movable returns list of marked objects to move into other shard. @@ -121,7 +122,7 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { }) }) if err != nil { - return MovableRes{}, err + return MovableRes{}, metaerr.Wrap(err) } // we can parse strings to structures in-place, but probably it seems @@ -132,8 +133,8 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { for i := range strAddrs { err = decodeAddressFromKey(&addrs[i], []byte(strAddrs[i])) if err != nil { - return MovableRes{}, fmt.Errorf("can't parse object address %v: %w", - strAddrs[i], err) + return MovableRes{}, metaerr.Wrap(fmt.Errorf("can't parse object address %v: %w", + strAddrs[i], err)) } } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 86830f7da..b7cff68dc 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -9,6 +9,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -83,7 +84,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { storagelog.OpField("metabase PUT")) } - return + return res, metaerr.Wrap(err) } func (db *DB) put(tx *bbolt.Tx, diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index b6b8f5b10..466476d4e 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -9,6 +9,7 @@ import ( v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -80,11 +81,11 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err currEpoch := db.epochState.CurrentEpoch() - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) return err - }) + })) } func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, currEpoch uint64) ([]oid.Address, error) { diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index fac8a079f..f60a4724d 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -1,6 +1,7 @@ package meta import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" ) @@ -28,7 +29,7 @@ func (db *DB) ReadShardID() ([]byte, error) { } return nil }) - return id, err + return id, metaerr.Wrap(err) } // WriteShardID writes shard it to db. @@ -42,11 +43,11 @@ func (db *DB) WriteShardID(id []byte) error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { b, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { return err } return b.Put(shardIDKey, id) - }) + })) } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index b53010a1e..6ba5a60cb 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -54,7 +55,7 @@ func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { @@ -113,5 +114,5 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e return err }) - return + return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 5b2c97f10..e7bcb110b 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -105,7 +106,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { func (t *boltForest) Open(readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { - return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err) + return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) } opts := *bbolt.DefaultOptions @@ -116,7 +117,7 @@ func (t *boltForest) Open(readOnly bool) error { t.db, err = bbolt.Open(t.path, t.perm, &opts) if err != nil { - return fmt.Errorf("can't open the pilorama DB: %w", err) + return metaerr.Wrap(fmt.Errorf("can't open the pilorama DB: %w", err)) } t.db.MaxBatchSize = t.maxBatchSize @@ -174,7 +175,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin lm := *m fullID := bucketName(d.CID, treeID) - return &lm, t.db.Batch(func(tx *bbolt.Tx) error { + return &lm, metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -185,7 +186,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin lm.Child = t.findSpareID(bTree) } return t.do(bLog, bTree, make([]byte, 17), &lm) - }) + })) } func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { @@ -219,7 +220,7 @@ func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID strin if err == nil { err = retErr } - return height, err + return height, metaerr.Wrap(err) } // TreeExists implements the Forest interface. @@ -247,7 +248,7 @@ func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID strin return nil }) - return exists, err + return exists, metaerr.Wrap(err) } var syncHeightKey = []byte{'h'} @@ -267,7 +268,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID binary.LittleEndian.PutUint64(rawHeight, height) buck := bucketName(cid, treeID) - return t.db.Batch(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(buck) if treeRoot == nil { return ErrTreeNotFound @@ -275,7 +276,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID b := treeRoot.Bucket(dataBucket) return b.Put(syncHeightKey, rawHeight) - }) + })) } // TreeLastSyncHeight implements the pilorama.Forest interface. @@ -304,7 +305,7 @@ func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, tree } return nil }) - return height, err + return height, metaerr.Wrap(err) } // TreeAddByPath implements the Forest interface. @@ -384,7 +385,7 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID } return t.do(bLog, bTree, key[:], &lm[len(lm)-1]) }) - return lm, err + return lm, metaerr.Wrap(err) } // getLatestTimestamp returns timestamp for a new operation which is guaranteed to be bigger than @@ -450,13 +451,13 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string return nil }) if err != nil || seen { - return err + return metaerr.Wrap(err) } } if t.db.MaxBatchSize == 1 { fullID := bucketName(cnr, treeID) - return t.db.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -464,12 +465,12 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var lm Move return t.applyOperation(bLog, bTree, []*Move{m}, &lm) - }) + })) } ch := make(chan error, 1) t.addBatch(cnr, treeID, m, ch) - return <-ch + return metaerr.Wrap(<-ch) } func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { @@ -751,7 +752,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st var nodes []Node - return nodes, t.db.View(func(tx *bbolt.Tx) error { + return nodes, metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { return ErrTreeNotFound @@ -788,7 +789,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st childKey, _ = c.Next() } return nil - }) + })) } // TreeGetMeta implements the forest interface. @@ -828,7 +829,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri return m.FromBytes(meta) }) - return m, parentID, err + return m, parentID, metaerr.Wrap(err) } // TreeGetChildren implements the Forest interface. @@ -869,7 +870,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID return nil }) - return children, err + return children, metaerr.Wrap(err) } // TreeList implements the Forest interface. @@ -907,7 +908,7 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err return nil }) if err != nil { - return nil, fmt.Errorf("could not list trees: %w", err) + return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) } return ids, nil @@ -949,7 +950,7 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str return nil }) - return lm, err + return lm, metaerr.Wrap(err) } // TreeDrop implements the pilorama.Forest interface. @@ -971,7 +972,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrReadOnlyMode } - return t.db.Batch(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { if treeID == "" { c := tx.Cursor() prefix := make([]byte, 32) @@ -989,7 +990,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrTreeNotFound } return err - }) + })) } func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index 796fda623..aeab88b0b 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -77,5 +78,5 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { deleted = true } - return err + return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index da8b3a0fa..21f968b04 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -10,6 +10,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -193,7 +194,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { data, err := f() if err != nil { - c.reportFlushError("can't read a file", sAddr, err) + c.reportFlushError("can't read a file", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -203,7 +204,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var obj object.Object err = obj.Unmarshal(data) if err != nil { - c.reportFlushError("can't unmarshal an object", sAddr, err) + c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -314,7 +315,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) if err := addr.DecodeString(sa); err != nil { - c.reportFlushError("can't decode object address from the DB", sa, err) + c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } @@ -323,7 +324,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { var obj object.Object if err := obj.Unmarshal(data); err != nil { - c.reportFlushError("can't unmarshal an object from the DB", sa, err) + c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index aac5759ae..2546bada9 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -5,6 +5,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -28,7 +29,8 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() - return c.getInternal(ctx, saddr, addr) + obj, err := c.getInternal(ctx, saddr, addr) + return obj, metaerr.Wrap(err) } func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { @@ -71,7 +73,7 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, obj, err := c.getInternal(ctx, saddr, addr) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return obj.CutPayload(), nil @@ -95,5 +97,5 @@ func Get(db *bbolt.DB, key []byte) ([]byte, error) { value = slice.Copy(value) return nil }) - return value, err + return value, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 59ace93bd..5349c069c 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -18,7 +19,7 @@ var ErrNoDefaultBucket = errors.New("no default bucket") // // DB must not be nil and should be opened. func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { - return db.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) if b == nil { return ErrNoDefaultBucket @@ -34,5 +35,5 @@ func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { return f(addr) }) - }) + })) } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 61e81ec30..619b2bd26 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -72,7 +73,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro if err == nil { added = true } - return common.PutRes{}, err + return common.PutRes{}, metaerr.Wrap(err) } // putSmall persists small objects to the write-cache database and diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 6fa0e36de..962c9c39a 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -129,14 +130,14 @@ func (c *cache) DumpInfo() Info { func (c *cache) Open(readOnly bool) error { err := c.openStore(readOnly) if err != nil { - return err + return metaerr.Wrap(err) } // Opening after Close is done during maintenance mode, // thus we need to create a channel here. c.closeCh = make(chan struct{}) - return c.initCounters() + return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. -- 2.45.2 From 69df0d21c2d5f14a28d45951c32702ceec9da759 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 14:50:51 +0300 Subject: [PATCH 124/233] [#446] engine: Move to read-only on blobstor errors Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 36 +++++++++++-------- pkg/local_object_storage/engine/error_test.go | 4 +-- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 9967b2e2b..3061d6383 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -45,6 +46,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -70,7 +72,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -82,7 +84,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -91,21 +93,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { e.mtx.RLock() defer e.mtx.RUnlock() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return + } log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Error(err)) - } else { - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) - } - } else { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -169,11 +173,13 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } + isMeta := errors.As(err, new(metaerr.Error)) if block { - e.moveToDegraded(sh, errCount) + e.moveToDegraded(sh, errCount, isMeta) } else { req := setModeRequest{ errorCount: errCount, + isMeta: isMeta, sh: sh, } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index dc28d35fa..18bc72d65 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } @@ -219,7 +219,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, te.ng, te.shards[0].id, 1, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } -- 2.45.2 From 50caa388b0467b9011650e38930167c7891093cd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Jun 2023 11:55:06 +0300 Subject: [PATCH 125/233] [#303] ir: Use pub key when validate container deletion Signed-off-by: Anton Nikiforov --- pkg/core/container/delete.go | 46 ++++--------------- .../processors/container/process_container.go | 1 + pkg/morph/client/container/delete.go | 15 ++++-- pkg/morph/event/container/delete.go | 3 +- pkg/morph/event/container/delete_notary.go | 5 ++ pkg/services/container/morph/executor.go | 8 ++-- 6 files changed, 31 insertions(+), 47 deletions(-) diff --git a/pkg/core/container/delete.go b/pkg/core/container/delete.go index e3379446f..8e0aaebb9 100644 --- a/pkg/core/container/delete.go +++ b/pkg/core/container/delete.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -8,43 +9,14 @@ import ( // RemovalWitness groups the information required // to prove and verify the removal of a container. type RemovalWitness struct { - cnr cid.ID + // ContainerID returns the identifier of the container + // to be removed. + ContainerID cid.ID - sig []byte + // Signature the signature of the container identifier. + Signature *refs.Signature - token *session.Container -} - -// ContainerID returns the identifier of the container -// to be removed. -func (x RemovalWitness) ContainerID() cid.ID { - return x.cnr -} - -// SetContainerID sets the identifier of the container -// to be removed. -func (x *RemovalWitness) SetContainerID(id cid.ID) { - x.cnr = id -} - -// Signature returns the signature of the container identifier. -func (x RemovalWitness) Signature() []byte { - return x.sig -} - -// SetSignature sets a signature of the container identifier. -func (x *RemovalWitness) SetSignature(sig []byte) { - x.sig = sig -} - -// SessionToken returns the token of the session within -// which the container was removed. -func (x RemovalWitness) SessionToken() *session.Container { - return x.token -} - -// SetSessionToken sets the token of the session within -// which the container was removed. -func (x *RemovalWitness) SetSessionToken(tok *session.Container) { - x.token = tok + // SessionToken the token of the session within + // which the container was removed. + SessionToken *session.Container } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 33ef90034..2629b9d29 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -148,6 +148,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { binTokenSession: e.SessionToken(), signature: e.Signature(), signedData: binCnr, + binPublicKey: e.PublicKeyValue, }) if err != nil { return fmt.Errorf("auth container removal: %w", err) diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c9105a3ca..5bc8fc188 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -14,14 +14,15 @@ import ( // Returns error if container ID is nil. func Delete(c *Client, witness core.RemovalWitness) error { binCnr := make([]byte, sha256.Size) - witness.ContainerID().Encode(binCnr) + witness.ContainerID.Encode(binCnr) var prm DeletePrm prm.SetCID(binCnr) - prm.SetSignature(witness.Signature()) + prm.SetSignature(witness.Signature.GetSign()) + prm.SetKey(witness.Signature.GetKey()) - if tok := witness.SessionToken(); tok != nil { + if tok := witness.SessionToken; tok != nil { prm.SetToken(tok.Marshal()) } @@ -33,6 +34,7 @@ type DeletePrm struct { cnr []byte signature []byte token []byte + key []byte client.InvokePrmOptional } @@ -52,6 +54,11 @@ func (d *DeletePrm) SetToken(token []byte) { d.token = token } +// SetKey sets public key. +func (d *DeletePrm) SetKey(key []byte) { + d.key = key +} + // Delete removes the container from FrostFS system // through Container contract call. // @@ -66,7 +73,7 @@ func (c *Client) Delete(p DeletePrm) error { prm := client.InvokePrm{} prm.SetMethod(deleteMethod) - prm.SetArgs(p.cnr, p.signature, p.token) + prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional err := c.client.Invoke(prm) diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 4926af27d..a206307f8 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -15,6 +15,7 @@ type Delete struct { ContainerIDValue []byte SignatureValue []byte TokenValue []byte + PublicKeyValue []byte // For notary notifications only. // Contains raw transactions of notary request. @@ -42,7 +43,7 @@ func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { return d.NotaryRequestValue } -const expectedItemNumDelete = 3 +const expectedItemNumDelete = 4 // DeleteSuccess structures notification event of successful container removal // thrown by Container contract. diff --git a/pkg/morph/event/container/delete_notary.go b/pkg/morph/event/container/delete_notary.go index 23f13acbb..9711636e7 100644 --- a/pkg/morph/event/container/delete_notary.go +++ b/pkg/morph/event/container/delete_notary.go @@ -17,6 +17,10 @@ func (d *Delete) setSignature(v []byte) { } } +func (d *Delete) setPublicKey(v []byte) { + d.PublicKeyValue = v +} + func (d *Delete) setToken(v []byte) { if v != nil { d.TokenValue = v @@ -26,6 +30,7 @@ func (d *Delete) setToken(v []byte) { var deleteFieldSetters = []func(*Delete, []byte){ // order on stack is reversed (*Delete).setToken, + (*Delete).setPublicKey, (*Delete).setSignature, (*Delete).setContainerID, } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 8e6b30856..ae37da520 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -109,8 +109,6 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * return nil, fmt.Errorf("invalid container ID: %w", err) } - sig := body.GetSignature().GetSign() - var tok *session.Container if tokV2 != nil { @@ -124,9 +122,9 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * var rmWitness containercore.RemovalWitness - rmWitness.SetContainerID(id) - rmWitness.SetSignature(sig) - rmWitness.SetSessionToken(tok) + rmWitness.ContainerID = id + rmWitness.Signature = body.GetSignature() + rmWitness.SessionToken = tok err = s.wrt.Delete(rmWitness) if err != nil { -- 2.45.2 From 01a0c977608ab6008d491b68097213d1265d309b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:59:18 +0300 Subject: [PATCH 126/233] [#453] engine: Set Disabled mode to deleted shard Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/shards.go | 9 ++++++++- pkg/local_object_storage/shard/errors.go | 3 +++ pkg/local_object_storage/shard/exists.go | 5 ++++- pkg/local_object_storage/shard/get.go | 5 +++++ pkg/local_object_storage/shard/mode.go | 8 +++++--- pkg/local_object_storage/shard/mode/mode.go | 4 ++++ pkg/local_object_storage/shard/range.go | 5 +++++ pkg/local_object_storage/shard/tree.go | 7 +++++++ 9 files changed, 42 insertions(+), 5 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dc54f2d2a..2ba9e6e4b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -492,4 +492,5 @@ const ( ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" + EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" ) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index cfa64929b..e80d7a158 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -203,7 +203,14 @@ func (e *StorageEngine) removeShards(ids ...string) { e.mtx.Unlock() for _, sh := range ss { - err := sh.Close() + err := sh.SetMode(mode.Disabled) + if err != nil { + e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + zap.Stringer("id", sh.ID()), + zap.Error(err), + ) + } + err = sh.Close() if err != nil { e.log.Error(logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), diff --git a/pkg/local_object_storage/shard/errors.go b/pkg/local_object_storage/shard/errors.go index 3e5224eb9..2958a492c 100644 --- a/pkg/local_object_storage/shard/errors.go +++ b/pkg/local_object_storage/shard/errors.go @@ -4,9 +4,12 @@ import ( "errors" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +var ErrShardDisabled = logicerr.New("shard disabled") + // IsErrNotFound checks if error returned by Shard Get/Head/GetRange method // corresponds to missing object. func IsErrNotFound(err error) bool { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 60809da6a..2cdb8dfa8 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -38,6 +38,7 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", trace.WithAttributes( @@ -52,7 +53,9 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { s.m.RLock() defer s.m.RUnlock() - if s.info.Mode.NoMetabase() { + if s.info.Mode.Disabled() { + return ExistsRes{}, ErrShardDisabled + } else if s.info.Mode.NoMetabase() { var p common.ExistsPrm p.Address = prm.addr diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 589ec53c9..3eb70784e 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -66,6 +66,7 @@ func (r GetRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in shard. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Get", trace.WithAttributes( @@ -78,6 +79,10 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.Mode.Disabled() { + return GetRes{}, ErrShardDisabled + } + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getPrm common.GetPrm getPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index a59f08704..1bab57448 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -56,9 +56,11 @@ func (s *Shard) setMode(m mode.Mode) error { } } - for i := range components { - if err := components[i].SetMode(m); err != nil { - return err + if !m.Disabled() { + for i := range components { + if err := components[i].SetMode(m); err != nil { + return err + } } } diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 65b2b5c89..49c888d63 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -57,3 +57,7 @@ func (m Mode) NoMetabase() bool { func (m Mode) ReadOnly() bool { return m&ReadOnly != 0 } + +func (m Mode) Disabled() bool { + return m == Disabled +} diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 895dc2fe0..362fd86c5 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -72,6 +72,7 @@ func (r RngRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", trace.WithAttributes( @@ -86,6 +87,10 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.Mode.Disabled() { + return RngRes{}, ErrShardDisabled + } + cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 2331d37a8..81477325a 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -265,6 +265,13 @@ func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (u ) defer span.End() + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return 0, ErrDegradedMode + } + if s.pilorama == nil { return 0, ErrPiloramaDisabled } -- 2.45.2 From 028d4a805859bcd338446ced8674d02fb98018b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:23:17 +0300 Subject: [PATCH 127/233] [#373] blobovnicza: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 10 ++++ .../blobovnicza/control.go | 14 +++++- .../blobovnicza/metrics.go | 16 +++++++ pkg/local_object_storage/blobovnicza/sizes.go | 2 + pkg/local_object_storage/engine/shards.go | 48 +++++++++++-------- 5 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/metrics.go diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index 21c9f9e58..ecd1dc5e5 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -31,6 +31,8 @@ type cfg struct { objSizeLimit uint64 log *logger.Logger + + metrics Metrics } type boltDBCfg struct { @@ -52,6 +54,7 @@ func defaultCfg(c *cfg) { fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB log: &logger.Logger{Logger: zap.L()}, + metrics: &noopMetrics{}, } } @@ -112,3 +115,10 @@ func WithReadOnly(ro bool) Option { c.boltOptions.ReadOnly = ro } } + +// WithMetrics returns an option to set metrics storage. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 84274528a..c776afe06 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -35,6 +35,9 @@ func (b *Blobovnicza) Open() error { ) b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) + if err == nil { + b.metrics.IncOpenCount() + } return err } @@ -81,7 +84,9 @@ func (b *Blobovnicza) Init() error { return fmt.Errorf("can't determine DB size: %w", err) } - b.filled.Store(uint64(info.Size())) + sz := uint64(info.Size()) + b.filled.Store(sz) + b.metrics.IncSize(sz) return err } @@ -91,5 +96,10 @@ func (b *Blobovnicza) Close() error { zap.String("path", b.path), ) - return b.boltDB.Close() + err := b.boltDB.Close() + if err == nil { + b.metrics.DecOpenCount() + b.metrics.DecSize(b.filled.Load()) + } + return err } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go new file mode 100644 index 000000000..1ffb7b1e2 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -0,0 +1,16 @@ +package blobovnicza + +type Metrics interface { + IncOpenCount() + DecOpenCount() + + IncSize(size uint64) + DecSize(size uint64) +} + +type noopMetrics struct{} + +func (m *noopMetrics) IncOpenCount() {} +func (m *noopMetrics) DecOpenCount() {} +func (m *noopMetrics) IncSize(uint64) {} +func (m *noopMetrics) DecSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 1cc100d19..bdbc77d18 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,10 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) + b.metrics.IncSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.filled.Add(^(sz - 1)) + b.metrics.DecSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index e80d7a158..f362e2a03 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -102,15 +102,35 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { return nil, fmt.Errorf("could not generate shard ID: %w", err) } + opts = e.appendMetrics(id, opts) + + sh := shard.New(append(opts, + shard.WithID(id), + shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), + shard.WithExpiredLocksCallback(e.processExpiredLocks), + shard.WithDeletedLockCallback(e.processDeletedLocks), + shard.WithReportErrorFunc(e.reportShardErrorBackground), + )...) + + if err := sh.UpdateID(); err != nil { + return nil, fmt.Errorf("could not update shard ID: %w", err) + } + + return sh, err +} + +func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard.Option { e.mtx.RLock() + defer e.mtx.RUnlock() if e.metrics != nil { - opts = append(opts, shard.WithMetricsWriter( - &metricsWithID{ - id: id.String(), - mw: e.metrics, - }, - ), + opts = append(opts, + shard.WithMetricsWriter( + &metricsWithID{ + id: id.String(), + mw: e.metrics, + }, + ), shard.WithExtraWriteCacheOptions(writecache.WithMetrics( &writeCacheMetrics{ shardID: id.String(), @@ -126,21 +146,7 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { ) } - e.mtx.RUnlock() - - sh := shard.New(append(opts, - shard.WithID(id), - shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), - shard.WithExpiredLocksCallback(e.processExpiredLocks), - shard.WithDeletedLockCallback(e.processDeletedLocks), - shard.WithReportErrorFunc(e.reportShardErrorBackground), - )...) - - if err := sh.UpdateID(); err != nil { - return nil, fmt.Errorf("could not update shard ID: %w", err) - } - - return sh, err + return opts } func (e *StorageEngine) addShard(sh *shard.Shard) error { -- 2.45.2 From a8526d45e9451275939e894ed4464d50356843ed Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:24:14 +0300 Subject: [PATCH 128/233] [#373] blobovnizca: Add missed/fix tracing spans Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovnicza/delete.go | 12 ++++++------ .../blobovnicza/exists.go | 19 ++++++++++++++++--- pkg/local_object_storage/blobovnicza/get.go | 1 + .../blobovnicza/get_test.go | 6 +++--- .../blobovnicza/iterate.go | 12 ++++++++++++ .../blobovnicza/iterate_test.go | 2 +- pkg/local_object_storage/blobovnicza/put.go | 14 +++++++++++++- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/put.go | 8 ++++---- 10 files changed, 58 insertions(+), 20 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 5deaf5e4a..05f8906ac 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -21,7 +21,7 @@ func testPutGet(t *testing.T, blz *Blobovnicza, addr oid.Address, sz uint64, ass var pPut PutPrm pPut.SetAddress(addr) pPut.SetMarshaledObject(data) - _, err := blz.Put(pPut) + _, err := blz.Put(context.Background(), pPut) if assertErrPut != nil { require.True(t, assertErrPut(err)) } else { diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index e880815a6..8fbd363db 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -38,13 +38,14 @@ func (p *DeletePrm) SetAddress(addr oid.Address) { func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Delete", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() addrKey := addressKey(prm.addr) - removed := false + found := false err := b.boltDB.Update(func(tx *bbolt.Tx) error { return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { @@ -56,9 +57,6 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err sz := uint64(len(objData)) - // decrease fullness counter - b.decSize(sz) - // remove object from the bucket err := buck.Delete(addrKey) @@ -67,16 +65,18 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err zap.String("binary size", stringifyByteSize(sz)), zap.String("range", stringifyBounds(lower, upper)), ) + // decrease fullness counter + b.decSize(sz) } - removed = true + found = true // stop iteration return true, err }) }) - if err == nil && !removed { + if err == nil && !found { var errNotFound apistatus.ObjectNotFound return DeleteRes{}, errNotFound diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 8ac45c4aa..e6d28f938 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,17 +1,30 @@ package blobovnicza import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Exists check if object with the specified address is stored in b. -func (b *Blobovnicza) Exists(addr oid.Address) (bool, error) { +func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error) { var ( - exists bool - addrKey = addressKey(addr) + exists = false ) + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Exists", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + addrKey := addressKey(addr) + err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error { exists = buck.Get(addrKey) != nil diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index ff29358b0..d492b7559 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -46,6 +46,7 @@ var errInterruptForEach = errors.New("interrupt for-each") func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Get", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index ad30e8d94..40c434eb7 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -41,7 +41,7 @@ func TestBlobovnicza_Get(t *testing.T) { addr := oidtest.Address() obj := make([]byte, firstBucketBound+1) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.False(t, exists) @@ -50,7 +50,7 @@ func TestBlobovnicza_Get(t *testing.T) { prmPut.SetMarshaledObject(obj) // place object to [32K:64K] bucket - _, err = blz.Put(prmPut) + _, err = blz.Put(context.Background(), prmPut) require.NoError(t, err) var prmGet GetPrm @@ -61,7 +61,7 @@ func TestBlobovnicza_Get(t *testing.T) { require.NoError(t, err) require.Equal(t, obj, res.Object()) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.True(t, exists) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 8f8357053..c2031ea54 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -4,8 +4,11 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) func (b *Blobovnicza) iterateBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { @@ -119,6 +122,15 @@ type IterateRes struct { // // Handler should not retain object data. Handler must not be nil. func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Iterate", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.Bool("decode_addresses", prm.decodeAddresses), + attribute.Bool("without_data", prm.withoutData), + attribute.Bool("ignore_errors", prm.ignoreErrors), + )) + defer span.End() + var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 505685ced..90308723c 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -20,7 +20,7 @@ func TestBlobovniczaIterate(t *testing.T) { data := [][]byte{{0, 1, 2, 3}, {5, 6, 7, 8}} addr := oidtest.Address() - _, err := b.Put(PutPrm{addr: addr, objData: data[0]}) + _, err := b.Put(context.Background(), PutPrm{addr: addr, objData: data[0]}) require.NoError(t, err) require.NoError(t, b.boltDB.Update(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 37ed57e1c..cc816b3e1 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -1,11 +1,15 @@ package blobovnicza import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // PutPrm groups the parameters of Put operation. @@ -47,7 +51,15 @@ func (p *PutPrm) SetMarshaledObject(data []byte) { // Returns ErrFull if blobovnicza is filled. // // Should not be called in read-only configuration. -func (b *Blobovnicza) Put(prm PutPrm) (PutRes, error) { +func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Put", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Int("size", len(prm.objData)), + )) + defer span.End() + sz := uint64(len(prm.objData)) bucketName := bucketForSize(sz) key := addressKey(prm.addr) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 3324507cf..c2a4740d8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -30,7 +30,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return common.ExistsRes{}, err } - exists, err := blz.Exists(prm.Address) + exists, err := blz.Exists(ctx, prm.Address) return common.ExistsRes{Exists: exists}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 95ed15540..a567baa3a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -45,7 +45,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe PutPrm: putPrm, } - if err := b.iterateDeepest(ctx, prm.Address, it.iterate); err != nil { + if err := b.iterateDeepest(ctx, prm.Address, func(s string) (bool, error) { return it.iterate(ctx, s) }); err != nil { return common.PutRes{}, err } else if it.ID == nil { if it.AllFull { @@ -64,7 +64,7 @@ type putIterator struct { PutPrm blobovnicza.PutPrm } -func (i *putIterator) iterate(path string) (bool, error) { +func (i *putIterator) iterate(ctx context.Context, path string) (bool, error) { active, err := i.B.getActivated(path) if err != nil { if !isLogical(err) { @@ -77,7 +77,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - if _, err := active.blz.Put(i.PutPrm); err != nil { + if _, err := active.blz.Put(ctx, i.PutPrm); err != nil { // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error // or update active blobovnicza in other thread. In the latter case the database will be closed // and `updateActive` takes care of not updating the active blobovnicza twice. @@ -99,7 +99,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - return i.iterate(path) + return i.iterate(ctx, path) } i.AllFull = false -- 2.45.2 From 3ae3c8dfdb806efc33ee809c512be5bce157a1ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Jun 2023 16:04:02 +0300 Subject: [PATCH 129/233] [#373] fstree: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/control.go | 6 +- .../blobstor/fstree/fstree.go | 100 ++++++++++++++++-- .../blobstor/fstree/metrics.go | 26 +++++ .../blobstor/fstree/option.go | 6 ++ 4 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/metrics.go diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index 1ff74893d..f41b7aacd 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -7,6 +7,7 @@ import ( // Open implements common.Storage. func (t *FSTree) Open(ro bool) error { t.readOnly = ro + t.metrics.SetMode(ro) return nil } @@ -16,4 +17,7 @@ func (t *FSTree) Init() error { } // Close implements common.Storage. -func (*FSTree) Close() error { return nil } +func (t *FSTree) Close() error { + t.metrics.Close() + return nil +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 76487813e..645d4e5f7 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "syscall" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -35,6 +36,7 @@ type FSTree struct { noSync bool readOnly bool + metrics Metrics } // Info groups the information about file storage. @@ -64,6 +66,7 @@ func New(opts ...Option) *FSTree { Config: nil, Depth: 4, DirNameLen: DirNameLen, + metrics: &noopMetrics{}, } for i := range opts { opts[i](f) @@ -101,7 +104,24 @@ func addressFromString(s string) (oid.Address, error) { // Iterate iterates over all stored objects. func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, t.iterate(ctx, 0, []string{t.RootPath}, prm) + var ( + err error + startedAt = time.Now() + ) + + defer func() { + t.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Iterate", + trace.WithAttributes( + attribute.String("path", t.RootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = t.iterate(ctx, 0, []string{t.RootPath}, prm) + return common.IterateRes{}, err } func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { @@ -202,19 +222,29 @@ func (t *FSTree) treePath(addr oid.Address) string { // Delete removes the object with the specified address from the storage. func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + err error + startedAt = time.Now() + ) + defer func() { + t.metrics.Delete(time.Since(startedAt), err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() if t.readOnly { - return common.DeleteRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.DeleteRes{}, err } p := t.treePath(prm.Address) - err := os.Remove(p) + err = os.Remove(p) if err != nil && os.IsNotExist(err) { err = logicerr.Wrap(apistatus.ObjectNotFound{}) } @@ -224,8 +254,17 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + t.metrics.Exists(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Exists", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() @@ -237,27 +276,40 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist if os.IsNotExist(err) { err = nil } + success = err == nil return common.ExistsRes{Exists: found}, err } // Put puts an object in the storage. func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + size int + startedAt = time.Now() + err error + ) + defer func() { + t.metrics.Put(time.Since(startedAt), size, err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.Bool("dont_compress", prm.DontCompress), )) defer span.End() if t.readOnly { - return common.PutRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.PutRes{}, err } p := t.treePath(prm.Address) - if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { + if err = util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { if errors.Is(err, syscall.ENOSPC) { - return common.PutRes{}, common.ErrNoSpace + err = common.ErrNoSpace + return common.PutRes{}, err } return common.PutRes{}, err } @@ -287,17 +339,19 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err // to be so hecking simple. // In a very rare situation we can have multiple partially written copies on disk, // this will be fixed in another issue (we should remove garbage on start). + size = len(prm.RawData) const retryCount = 5 for i := 0; i < retryCount; i++ { tmpPath := p + "#" + strconv.FormatUint(uint64(i), 10) - err := t.writeAndRename(tmpPath, p, prm.RawData) + err = t.writeAndRename(tmpPath, p, prm.RawData) if err != syscall.EEXIST || i == retryCount-1 { return common.PutRes{StorageID: []byte{}}, err } } + err = fmt.Errorf("couldn't read file after %d retries", retryCount) // unreachable, but precaution never hurts, especially 1 day before release. - return common.PutRes{StorageID: []byte{}}, fmt.Errorf("couldn't read file after %d retries", retryCount) + return common.PutRes{StorageID: []byte{}}, err } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. @@ -365,8 +419,18 @@ func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.Get(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.Get", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.Bool("raw", prm.Raw), attribute.String("address", prm.Address.EncodeToString()), )) @@ -394,19 +458,30 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err if err != nil { return common.GetRes{}, err } + size = len(data) obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { return common.GetRes{}, err } - - return common.GetRes{Object: obj, RawData: data}, err + success = true + return common.GetRes{Object: obj, RawData: data}, nil } // GetRange implements common.Storage. func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.GetRange(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.GetRange", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), @@ -426,8 +501,11 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } + success = true + data := payload[from:to] + size = len(data) return common.GetRangeRes{ - Data: payload[from:to], + Data: data, }, nil } diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go new file mode 100644 index 000000000..04386d9b8 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -0,0 +1,26 @@ +package fstree + +import "time" + +type Metrics interface { + SetMode(readOnly bool) + Close() + + Iterate(d time.Duration, success bool) + Delete(d time.Duration, success bool) + Exists(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) + Get(d time.Duration, size int, success bool) + GetRange(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Delete(time.Duration, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 07e547444..52c8718c2 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -35,3 +35,9 @@ func WithNoSync(noSync bool) Option { f.noSync = noSync } } + +func WithMetrics(m Metrics) Option { + return func(f *FSTree) { + f.metrics = m + } +} -- 2.45.2 From 8318d90ad0c34847d4ea0d2576f44f97fe6148ac Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Jun 2023 10:25:25 +0300 Subject: [PATCH 130/233] [#373] blobovniczatree: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 2 +- .../blobovnicza/metrics.go | 10 +++--- .../blobstor/blobovniczatree/control.go | 7 +++- .../blobstor/blobovniczatree/delete.go | 16 ++++++++- .../blobstor/blobovniczatree/exists.go | 13 ++++++- .../blobstor/blobovniczatree/get.go | 20 ++++++++++- .../blobstor/blobovniczatree/get_range.go | 27 +++++++++++--- .../blobstor/blobovniczatree/iterate.go | 22 +++++++++++- .../blobstor/blobovniczatree/metrics.go | 35 +++++++++++++++++++ .../blobstor/blobovniczatree/option.go | 8 +++++ .../blobstor/blobovniczatree/put.go | 12 +++++++ 11 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/metrics.go diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index ecd1dc5e5..d5741fba7 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -54,7 +54,7 @@ func defaultCfg(c *cfg) { fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB log: &logger.Logger{Logger: zap.L()}, - metrics: &noopMetrics{}, + metrics: &NoopMetrics{}, } } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 1ffb7b1e2..6127370bc 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -8,9 +8,9 @@ type Metrics interface { DecSize(size uint64) } -type noopMetrics struct{} +type NoopMetrics struct{} -func (m *noopMetrics) IncOpenCount() {} -func (m *noopMetrics) DecOpenCount() {} -func (m *noopMetrics) IncSize(uint64) {} -func (m *noopMetrics) DecSize(uint64) {} +func (m *NoopMetrics) IncOpenCount() {} +func (m *NoopMetrics) DecOpenCount() {} +func (m *NoopMetrics) IncSize(uint64) {} +func (m *NoopMetrics) DecSize(uint64) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0b2135631..0045e08db 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -13,6 +13,7 @@ import ( // Open opens blobovnicza tree. func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly + b.metrics.SetMode(readOnly) return nil } @@ -70,6 +71,7 @@ func (b *Blobovniczas) Close() error { } b.active = make(map[string]blobovniczaWithIndex) + b.metrics.Close() b.lruMtx.Unlock() @@ -123,9 +125,12 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz b.openMtx.Lock() defer b.openMtx.Unlock() + path := filepath.Join(b.rootPath, p) + blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), - blobovnicza.WithPath(filepath.Join(b.rootPath, p)), + blobovnicza.WithPath(path), + blobovnicza.WithMetrics(b.metrics.BlobovnicaMetrics(path)), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 1e38c6ea1..4d0801ef1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -21,8 +22,17 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res common.DeleteRes, err error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Delete", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -42,7 +52,10 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return res, err } - return b.deleteObject(ctx, blz, bPrm) + if res, err = b.deleteObject(ctx, blz, bPrm); err == nil { + success = true + } + return res, err } activeCache := make(map[string]struct{}) @@ -78,6 +91,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // not found in any blobovnicza return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + success = err == nil return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index c2a4740d8..cd553ec30 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -16,8 +17,18 @@ import ( // Exists implements common.Storage. func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + startedAt = time.Now() + success = false + found = false + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Exists", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -39,7 +50,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) - var found bool err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -59,5 +69,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return found, nil }) + success = err == nil return common.ExistsRes{Exists: found}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 3c8e288d4..300e3ce3a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -23,8 +24,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + found = false + size = 0 + ) + defer func() { + b.metrics.Get(time.Since(startedAt), size, found, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.Bool("raw", prm.Raw), @@ -41,7 +52,11 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, err } - return b.getObject(ctx, blz, bPrm) + res, err = b.getObject(ctx, blz, bPrm) + if err == nil { + found = true + size = len(res.RawData) + } } activeCache := make(map[string]struct{}) @@ -72,6 +87,9 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, logicerr.Wrap(apistatus.ObjectNotFound{}) } + found = true + size = len(res.RawData) + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 6e9620adf..3530c1a41 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -24,8 +25,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + found = false + size = 0 + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), size, found, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), @@ -40,11 +51,15 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re return common.GetRangeRes{}, err } - return b.getObjectRange(ctx, blz, prm) + res, err := b.getObjectRange(ctx, blz, prm) + if err == nil { + size = len(res.Data) + found = true + } + return res, err } activeCache := make(map[string]struct{}) - objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -67,17 +82,21 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache[dirPath] = struct{}{} - objectFound = err == nil + found = err == nil // abort iterator if found, otherwise process all Blobovniczas return err == nil, nil }) - if err == nil && !objectFound { + if err == nil && !found { // not found in any blobovnicza return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + if err == nil { + size = len(res.Data) + } + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 140716690..0154fe2ca 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -4,16 +4,35 @@ import ( "context" "fmt" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Iterate iterates over all objects in b. func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { + var ( + startedAt = time.Now() + err error + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Iterate", + trace.WithAttributes( + attribute.String("path", b.rootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { data, err := b.compression.Decompress(elem.ObjectData()) @@ -43,6 +62,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm _, err := blz.Iterate(ctx, subPrm) return err }) + return common.IterateRes{}, err } // iterator over all Blobovniczas in unsorted order. Break on f's error return. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go new file mode 100644 index 000000000..ffa3c8a8b --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -0,0 +1,35 @@ +package blobovniczatree + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" +) + +type Metrics interface { + BlobovnicaMetrics(path string) blobovnicza.Metrics + + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 95ef8635a..d0503f23b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -21,6 +21,7 @@ type cfg struct { blzOpts []blobovnicza.Option // reportError is the function called when encountering disk errors. reportError func(string, error) + metrics Metrics } type Option func(*cfg) @@ -40,6 +41,7 @@ func initConfig(c *cfg) { blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, + metrics: &noopMetrics{}, } } @@ -91,3 +93,9 @@ func WithObjectSizeLimit(sz uint64) Option { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index a567baa3a..038d5244d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -4,6 +4,7 @@ import ( "context" "errors" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -19,6 +20,15 @@ import ( // // returns error if could not save object in any blobovnicza. func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + success bool + size int + startedAt = time.Now() + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -33,6 +43,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe if !prm.DontCompress { prm.RawData = b.compression.Compress(prm.RawData) } + size = len(prm.RawData) var putPrm blobovnicza.PutPrm putPrm.SetAddress(prm.Address) @@ -54,6 +65,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe return common.PutRes{}, errPutFailed } + success = true return common.PutRes{StorageID: it.ID.Bytes()}, nil } -- 2.45.2 From f54cc0b607a3f02b167c79d7bc09bd67d8442878 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Jun 2023 09:05:52 +0300 Subject: [PATCH 131/233] [#373] blobstor: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/get.go | 8 +++--- .../blobstor/blobovniczatree/get_range.go | 12 +++++---- pkg/local_object_storage/blobstor/blobstor.go | 8 ++++++ pkg/local_object_storage/blobstor/control.go | 4 +++ pkg/local_object_storage/blobstor/delete.go | 11 ++++++++ pkg/local_object_storage/blobstor/exists.go | 18 +++++++++++-- pkg/local_object_storage/blobstor/get.go | 18 ++++++++++--- .../blobstor/get_range.go | 18 ++++++++++--- pkg/local_object_storage/blobstor/iterate.go | 18 +++++++++++++ pkg/local_object_storage/blobstor/metrics.go | 26 +++++++++++++++++++ pkg/local_object_storage/blobstor/put.go | 12 +++++++++ 11 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/metrics.go diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 300e3ce3a..5dafedd1c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -26,11 +26,11 @@ import ( func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { var ( startedAt = time.Now() - found = false + success = false size = 0 ) defer func() { - b.metrics.Get(time.Since(startedAt), size, found, prm.StorageID != nil) + b.metrics.Get(time.Since(startedAt), size, success, prm.StorageID != nil) }() ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", @@ -54,7 +54,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G res, err = b.getObject(ctx, blz, bPrm) if err == nil { - found = true + success = true size = len(res.RawData) } } @@ -87,7 +87,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, logicerr.Wrap(apistatus.ObjectNotFound{}) } - found = true + success = true size = len(res.RawData) return diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 3530c1a41..8579502c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -27,11 +27,11 @@ import ( func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { var ( startedAt = time.Now() - found = false + success = false size = 0 ) defer func() { - b.metrics.GetRange(time.Since(startedAt), size, found, prm.StorageID != nil) + b.metrics.GetRange(time.Since(startedAt), size, success, prm.StorageID != nil) }() ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", @@ -54,12 +54,13 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re res, err := b.getObjectRange(ctx, blz, prm) if err == nil { size = len(res.Data) - found = true + success = true } return res, err } activeCache := make(map[string]struct{}) + objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -82,18 +83,19 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache[dirPath] = struct{}{} - found = err == nil + objectFound = err == nil // abort iterator if found, otherwise process all Blobovniczas return err == nil, nil }) - if err == nil && !found { + if err == nil && !objectFound { // not found in any blobovnicza return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if err == nil { + success = true size = len(res.Data) } diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index a6fe9935e..a422b2bce 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -43,10 +43,12 @@ type cfg struct { compression compression.Config log *logger.Logger storage []SubStorage + metrics Metrics } func initConfig(c *cfg) { c.log = &logger.Logger{Logger: zap.L()} + c.metrics = &noopMetrics{} } // New creates, initializes and returns new BlobStor instance. @@ -113,3 +115,9 @@ func (b *BlobStor) SetReportErrorFunc(f func(string, error)) { b.storage[i].Storage.SetReportErrorFunc(f) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index abe39575b..6b439dcf0 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -18,6 +18,7 @@ func (b *BlobStor) Open(readOnly bool) error { return err } } + b.metrics.SetMode(readOnly) return nil } @@ -65,5 +66,8 @@ func (b *BlobStor) Close() error { if firstErr == nil { firstErr = err } + if firstErr == nil { + b.metrics.Close() + } return firstErr } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index f1b14481c..fe49fc46a 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -13,6 +14,14 @@ import ( ) func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -28,6 +37,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := b.storage[i].Storage.Delete(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) } return res, err @@ -45,6 +55,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := st.Delete(ctx, prm) if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 760e7b2a4..03dad392a 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "encoding/hex" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -17,6 +18,14 @@ import ( // Returns any error encountered that did not allow // to completely check object existence. func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + exists = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), exists, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Exists", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -29,9 +38,13 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi if prm.StorageID != nil { if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + res, err := b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } - return b.storage[0].Storage.Exists(ctx, prm) + res, err := b.storage[0].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } // If there was an error during existence check below, @@ -47,6 +60,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for i := range b.storage { res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { + exists = true return res, nil } else if err != nil { errors = append(errors, err) diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index eadb990c8..fab86ea5d 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -16,7 +17,14 @@ import ( // Get reads the object from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { +func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Get", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -30,7 +38,7 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Get(ctx, prm) + res, err = b.storage[i].Storage.Get(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -39,7 +47,9 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + } else { + res, err = b.storage[0].Storage.Get(ctx, prm) } - return b.storage[0].Storage.Get(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index ca4e41f33..671fbf02e 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -17,7 +18,14 @@ import ( // GetRange reads object payload data from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { +func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.GetRange", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -32,7 +40,7 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.GetRange(ctx, prm) + res, err = b.storage[i].Storage.GetRange(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -41,7 +49,9 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + } else { + res, err = b.storage[0].Storage.GetRange(ctx, prm) } - return b.storage[0].Storage.GetRange(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 4e52f0abf..5a41e4c4f 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -3,10 +3,14 @@ package blobstor import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -18,6 +22,19 @@ import ( // // If handler returns an error, method wraps and returns it immediately. func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Iterate", + trace.WithAttributes( + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() @@ -27,6 +44,7 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } } + success = true return common.IterateRes{}, nil } diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go new file mode 100644 index 000000000..4fd6f3e54 --- /dev/null +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -0,0 +1,26 @@ +package blobstor + +import "time" + +type Metrics interface { + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 125b445b6..a748750dd 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -26,6 +27,15 @@ var ErrNoPlaceFound = logicerr.New("couldn't find a place to store an object") // Returns any error encountered that // did not allow to completely save the object. func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -47,11 +57,13 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e } prm.RawData = data } + size = len(prm.RawData) for i := range b.storage { if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { + success = true logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } return res, err -- 2.45.2 From 059e9e88a25ab86752629d12165c02a3c2c32f46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Jun 2023 12:27:19 +0300 Subject: [PATCH 132/233] [#373] metabase: Add metrics Signed-off-by: Dmitrii Stepanov --- .../internal/meta/list-garbage.go | 2 +- .../internal/meta/list-graveyard.go | 2 +- cmd/frostfs-node/container.go | 2 +- cmd/frostfs-node/notificator.go | 2 +- pkg/local_object_storage/engine/container.go | 14 +++--- pkg/local_object_storage/engine/evacuate.go | 2 +- .../engine/evacuate_test.go | 4 +- pkg/local_object_storage/engine/list.go | 5 +- pkg/local_object_storage/engine/list_test.go | 2 +- .../engine/remove_copies.go | 2 +- pkg/local_object_storage/engine/select.go | 12 ++--- pkg/local_object_storage/metabase/children.go | 24 +++++++++- .../metabase/containers.go | 18 +++++++- .../metabase/containers_test.go | 13 +++--- pkg/local_object_storage/metabase/control.go | 10 +++- pkg/local_object_storage/metabase/db.go | 2 + pkg/local_object_storage/metabase/delete.go | 10 ++++ pkg/local_object_storage/metabase/exists.go | 11 ++++- pkg/local_object_storage/metabase/expired.go | 20 ++++++++ pkg/local_object_storage/metabase/get.go | 11 ++++- .../metabase/graveyard.go | 46 +++++++++++++++++-- .../metabase/graveyard_test.go | 30 ++++++------ pkg/local_object_storage/metabase/inhume.go | 10 +++- .../metabase/iterators.go | 36 +++++++++++++-- .../metabase/iterators_test.go | 6 +-- pkg/local_object_storage/metabase/list.go | 24 +++++++++- .../metabase/list_test.go | 5 +- pkg/local_object_storage/metabase/lock.go | 42 ++++++++++++++--- pkg/local_object_storage/metabase/metrics.go | 20 ++++++++ pkg/local_object_storage/metabase/mode.go | 1 + pkg/local_object_storage/metabase/put.go | 10 ++++ pkg/local_object_storage/metabase/select.go | 12 ++++- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/gc.go | 14 +++--- pkg/local_object_storage/shard/list.go | 36 ++++++++++++--- pkg/local_object_storage/shard/list_test.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +- pkg/services/policer/process.go | 2 +- pkg/services/policer/queue.go | 5 +- 39 files changed, 379 insertions(+), 96 deletions(-) create mode 100644 pkg/local_object_storage/metabase/metrics.go diff --git a/cmd/frostfs-lens/internal/meta/list-garbage.go b/cmd/frostfs-lens/internal/meta/list-garbage.go index 3ab9a8f88..61b10ca1f 100644 --- a/cmd/frostfs-lens/internal/meta/list-garbage.go +++ b/cmd/frostfs-lens/internal/meta/list-garbage.go @@ -28,6 +28,6 @@ func listGarbageFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGarbage(garbPrm) + err := db.IterateOverGarbage(cmd.Context(), garbPrm) common.ExitOnErr(cmd, common.Errf("could not iterate over garbage bucket: %w", err)) } diff --git a/cmd/frostfs-lens/internal/meta/list-graveyard.go b/cmd/frostfs-lens/internal/meta/list-graveyard.go index db90513eb..19a93691c 100644 --- a/cmd/frostfs-lens/internal/meta/list-graveyard.go +++ b/cmd/frostfs-lens/internal/meta/list-graveyard.go @@ -33,6 +33,6 @@ func listGraveyardFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGraveyard(gravePrm) + err := db.IterateOverGraveyard(cmd.Context(), gravePrm) common.ExitOnErr(cmd, common.Errf("could not iterate over graveyard bucket: %w", err)) } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 569e4a7ca..d5a5afce1 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -452,7 +452,7 @@ type localStorageLoad struct { } func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error { - idList, err := engine.ListContainers(d.engine) + idList, err := engine.ListContainers(context.TODO(), d.engine) if err != nil { return fmt.Errorf("list containers on engine failure: %w", err) } diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 358b39a72..3fa486955 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -27,7 +27,7 @@ type notificationSource struct { func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) { log := n.l.With(zap.Uint64("epoch", epoch)) - listRes, err := n.e.ListContainers(engine.ListContainersPrm{}) + listRes, err := n.e.ListContainers(ctx, engine.ListContainersPrm{}) if err != nil { log.Error(logs.FrostFSNodeNotificatorCouldNotListContainers, zap.Error(err)) return diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 061e2fea0..e45f502ac 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -1,6 +1,8 @@ package engine import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" @@ -92,9 +94,9 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe // ListContainers returns a unique container IDs presented in the engine objects. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersRes, err error) { +func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) (res ListContainersRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.listContainers() + res, err = e.listContainers(ctx) return err }) @@ -102,10 +104,10 @@ func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersR } // ListContainers calls ListContainers method on engine to get a unique container IDs presented in the engine objects. -func ListContainers(e *StorageEngine) ([]cid.ID, error) { +func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { var prm ListContainersPrm - res, err := e.ListContainers(prm) + res, err := e.ListContainers(ctx, prm) if err != nil { return nil, err } @@ -113,7 +115,7 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers() (ListContainersRes, error) { +func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { if e.metrics != nil { defer elapsed("ListContainers", e.metrics.AddMethodDuration)() } @@ -121,7 +123,7 @@ func (e *StorageEngine) listContainers() (ListContainersRes, error) { uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Shard.ListContainers(shard.ListContainersPrm{}) + res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(sh, "can't get list of containers", err) return false diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 73b7a7830..98a3a202d 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -261,7 +261,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes // because ListWithCursor works only with the metabase. - listRes, err := sh.ListWithCursor(listPrm) + listRes, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 43737e7f7..13ac59f24 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -69,7 +69,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng err := e.Put(context.Background(), putPrm) require.NoError(t, err) - res, err := e.shards[ids[len(ids)-1].String()].List() + res, err := e.shards[ids[len(ids)-1].String()].List(context.Background()) require.NoError(t, err) if len(res.AddressList()) == objPerShard { break @@ -209,7 +209,7 @@ func TestEvacuateNetwork(t *testing.T) { var totalCount uint64 for i := range evacuateIDs { - res, err := e.shards[ids[i].String()].List() + res, err := e.shards[ids[i].String()].List(context.Background()) require.NoError(t, err) totalCount += uint64(len(res.AddressList())) diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 8781416f4..f9229a2b1 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -1,6 +1,7 @@ package engine import ( + "context" "math/rand" "sort" @@ -96,7 +97,7 @@ func (l ListWithCursorRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { result := make([]objectcore.AddressWithType, 0, prm.count) // Set initial cursors @@ -142,7 +143,7 @@ func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes shardPrm.WithCount(count) shardPrm.WithCursor(cursor.getCurrentShardCursor()) - res, err := shardInstance.ListWithCursor(shardPrm) + res, err := shardInstance.ListWithCursor(ctx, shardPrm) if err != nil { cursor.setShardRead(curr) continue diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 5b927cf11..6cea2d0f4 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -107,7 +107,7 @@ func TestListWithCursor(t *testing.T) { var prm ListWithCursorPrm prm.count = tt.batchSize for { - res, err := e.ListWithCursor(prm) + res, err := e.ListWithCursor(context.Background(), prm) if err == ErrEndOfListing { require.Empty(t, res.AddressList()) break diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 7681e0e50..4b48d179c 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -69,7 +69,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat var listPrm shard.ListWithCursorPrm listPrm.WithCount(uint32(prm.Concurrency)) listPrm.WithCursor(cursor) - res, err := sh.ListWithCursor(listPrm) + res, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) { return nil diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 9f651845f..6a7bf1b7f 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -98,16 +98,16 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, // If limit is zero, then returns all available object addresses. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) { +func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.list(limit) + res, err = e.list(ctx, limit) return err }) return } -func (e *StorageEngine) list(limit uint64) (SelectRes, error) { +func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { if e.metrics != nil { defer elapsed("ListObjects", e.metrics.AddMethodDuration)() } @@ -118,7 +118,7 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { // consider iterating over shuffled shards e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.List() // consider limit result of shard iterator + res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { e.reportShardError(sh, "could not select objects from shard", err) } else { @@ -159,8 +159,8 @@ func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.S // List returns `limit` available physically storage object addresses in // engine. If limit is zero, then returns all available object addresses. -func List(storage *StorageEngine, limit uint64) ([]oid.Address, error) { - res, err := storage.List(limit) +func List(ctx context.Context, storage *StorageEngine, limit uint64) ([]oid.Address, error) { + res, err := storage.List(ctx, limit) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go index 500e83e7a..6816358d2 100644 --- a/pkg/local_object_storage/metabase/children.go +++ b/pkg/local_object_storage/metabase/children.go @@ -1,14 +1,34 @@ package meta import ( + "context" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetChildren returns parent -> children map. // If an object has no children, then map will contain addr -> empty slice value. -func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Address, error) { +func (db *DB) GetChildren(ctx context.Context, addresses []oid.Address) (map[oid.Address][]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("GetChildren", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.GetChildren", + trace.WithAttributes( + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -53,6 +73,6 @@ func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Addres if err != nil { return nil, metaerr.Wrap(err) } - + success = true return result, nil } diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index fe38e0b6d..472b2affc 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -1,14 +1,28 @@ package meta import ( + "context" "encoding/binary" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) -func (db *DB) Containers() (list []cid.ID, err error) { +func (db *DB) Containers(ctx context.Context) (list []cid.ID, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Containers", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.Containers") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -21,7 +35,7 @@ func (db *DB) Containers() (list []cid.ID, err error) { return err }) - + success = err == nil return list, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index c0565b35a..4e2dd550d 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "math/rand" "sort" "testing" @@ -34,7 +35,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, err) } - lst, err := db.Containers() + lst, err := db.Containers(context.Background()) require.NoError(t, err) for _, cnr := range lst { @@ -60,7 +61,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() @@ -68,7 +69,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address())) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) @@ -78,14 +79,14 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() assertContains(cnrs, cnr) require.NoError(t, metaToMoveIt(db, object.AddressOf(obj))) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) @@ -126,7 +127,7 @@ func TestDB_ContainersCount(t *testing.T) { return expected[i].EncodeToString() < expected[j].EncodeToString() }) - got, err := db.Containers() + got, err := db.Containers(context.Background()) require.NoError(t, err) sort.Slice(got, func(i, j int) bool { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 370fddda1..d0a9c4723 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -175,10 +175,14 @@ func (db *DB) SyncCounters() error { // Close closes boltDB instance. func (db *DB) Close() error { + var err error if db.boltDB != nil { - return metaerr.Wrap(db.boltDB.Close()) + err = metaerr.Wrap(db.boltDB.Close()) } - return nil + if err == nil { + db.metrics.Close() + } + return err } // Reload reloads part of the configuration. @@ -202,12 +206,14 @@ func (db *DB) Reload(opts ...Option) (bool, error) { } db.mode = mode.Degraded + db.metrics.SetMode(mode.Degraded) db.info.Path = c.info.Path if err := db.openBolt(); err != nil { return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite + db.metrics.SetMode(mode.ReadWrite) return true, nil } diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 5a9ca3aa9..c54ed98d0 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -60,6 +60,7 @@ type cfg struct { log *logger.Logger epochState EpochState + metrics Metrics } func defaultCfg() *cfg { @@ -70,6 +71,7 @@ func defaultCfg() *cfg { boltBatchDelay: bbolt.DefaultMaxBatchDelay, boltBatchSize: bbolt.DefaultMaxBatchSize, log: &logger.Logger{Logger: zap.L()}, + metrics: &noopMetrics{}, } } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 7e8e0e5dd..d387b3d04 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" @@ -71,6 +72,14 @@ type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + var ( + startedAt = time.Now() + deleted = false + ) + defer func() { + db.metrics.AddMethodDuration("Delete", time.Since(startedAt), deleted) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", trace.WithAttributes( attribute.Int("addr_count", len(prm.addrs)), @@ -98,6 +107,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return err }) if err == nil { + deleted = true for i := range prm.addrs { storagelog.Write(db.log, storagelog.AddressField(prm.addrs[i]), diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index f9c563bdd..b6e5ea052 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" @@ -45,6 +46,14 @@ func (p ExistsRes) Exists() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Exists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Exists", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -65,7 +74,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index aac158ba4..43933d12d 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -5,18 +5,37 @@ import ( "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FilterExpired return expired items from addresses. // Address considered expired if metabase does contain information about expiration and // expiration epoch is less than epoch. func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = true + ) + defer func() { + db.metrics.AddMethodDuration("FilterExpired", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.FilterExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -68,6 +87,7 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A if err != nil { return nil, metaerr.Wrap(err) } + success = true return result, nil } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 2ce82bcee..ad35b4c18 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -52,6 +53,14 @@ func (r GetRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Get", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Get", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -74,7 +83,7 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index e2530bd31..df9a3d302 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -2,10 +2,13 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -58,7 +61,18 @@ func (g *GarbageIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { +func (db *DB) IterateOverGarbage(ctx context.Context, p GarbageIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGarbage", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGarbage") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -66,9 +80,11 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { return ErrDegradedMode } - return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, gcHandler{p.h}, p.offset) })) + success = err == nil + return err } // TombstonedObject represents descriptor of the @@ -125,7 +141,18 @@ func (g *GraveyardIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { +func (db *DB) IterateOverGraveyard(ctx context.Context, p GraveyardIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGraveyard", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGraveyard") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -232,7 +259,18 @@ func graveFromKV(k, v []byte) (res TombstonedObject, err error) { // graveyard bucket. // // Returns any error appeared during deletion process. -func (db *DB) DropGraves(tss []TombstonedObject) error { +func (db *DB) DropGraves(ctx context.Context, tss []TombstonedObject) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("DropGraves", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.DropGraves") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 8cd09e3f7..7476608f2 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -23,7 +23,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err := db.IterateOverGraveyard(iterGravePRM) + err := db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) @@ -33,7 +33,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) require.Zero(t, counter) } @@ -83,7 +83,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the second object would be put after the @@ -99,7 +99,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the third object would be put before the @@ -164,7 +164,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) var iterGCPRM meta.GarbageIterationPrm @@ -175,7 +175,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // objects covered with a tombstone @@ -255,7 +255,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGraveyard)) @@ -272,7 +272,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, len(expectedGraveyard), counter) require.ElementsMatch(t, gotGraveyard, expectedGraveyard) @@ -287,7 +287,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -348,7 +348,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGarbage)) @@ -363,7 +363,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, len(expectedGarbage), counter) require.ElementsMatch(t, gotGarbage, expectedGarbage) @@ -378,7 +378,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -418,11 +418,11 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Equal(t, 2, counter) - err = db.DropGraves(buriedTS) + err = db.DropGraves(context.Background(), buriedTS) require.NoError(t, err) counter = 0 @@ -431,7 +431,7 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 7865a8771..76744aa33 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -122,6 +123,13 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Inhume", time.Since(startedAt), success) + }() _, span := tracing.StartSpanFromContext(ctx, "metabase.Inhume") defer span.End() @@ -138,7 +146,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err err = db.boltDB.Update(func(tx *bbolt.Tx) error { return db.inhumeTx(tx, currEpoch, prm, &res) }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index dfa048621..78bfd2914 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -1,17 +1,22 @@ package meta import ( + "context" "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExpiredObject is a descriptor of expired object from DB. @@ -44,7 +49,20 @@ var ErrInterruptIterator = logicerr.New("iterator is interrupted") // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { +func (db *DB) IterateExpired(ctx context.Context, epoch uint64, h ExpiredObjectHandler) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateExpired", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -52,9 +70,11 @@ func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { return ErrDegradedMode } - return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateExpired(tx, epoch, h) })) + success = err == nil + return err } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { @@ -125,7 +145,17 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) // Returns other errors of h directly. // // Does not modify tss. -func (db *DB) IterateCoveredByTombstones(tss map[string]oid.Address, h func(oid.Address) error) error { +func (db *DB) IterateCoveredByTombstones(ctx context.Context, tss map[string]oid.Address, h func(oid.Address) error) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateCoveredByTombstones", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateCoveredByTombstones") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 69bf2bee5..e07184eb6 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -36,7 +36,7 @@ func TestDB_IterateExpired(t *testing.T) { require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) - err := db.IterateExpired(epoch, func(exp *meta.ExpiredObject) error { + err := db.IterateExpired(context.Background(), epoch, func(exp *meta.ExpiredObject) error { if addr, ok := mAlive[exp.Type()]; ok { require.NotEqual(t, addr, exp.Address()) } @@ -96,7 +96,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { ts.EncodeToString(): ts, } - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) @@ -112,7 +112,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { handled = handled[:0] - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 7c77cd703..337265318 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -1,13 +1,19 @@ package meta import ( + "context" + "time" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrEndOfListing is returned from object listing with cursor @@ -61,7 +67,21 @@ func (l ListRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { +func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ListWithCursor", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.ListWithCursor", + trace.WithAttributes( + attribute.Int("count", prm.count), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -75,7 +95,7 @@ func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { res.addrList, res.cursor, err = db.listWithCursor(tx, result, prm.count, prm.cursor) return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 4bf3ca827..abb55c9d1 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "sort" "testing" @@ -51,7 +52,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - res, err := db.ListWithCursor(prm) + res, err := db.ListWithCursor(context.Background(), prm) if err != nil { if err != meta.ErrEndOfListing { b.Fatalf("error: %v", err) @@ -225,6 +226,6 @@ func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]objec listPrm.SetCount(count) listPrm.SetCursor(cursor) - r, err := db.ListWithCursor(listPrm) + r, err := db.ListWithCursor(context.Background(), listPrm) return r.AddressList(), r.Cursor(), err } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index b84e0bf46..f3388daf0 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -36,6 +37,14 @@ func bucketNameLockers(idCnr cid.ID, key []byte) []byte { // // Locked list should be unique. Panics if it is empty. func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid.ID) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Lock", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Lock", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -57,7 +66,12 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. panic("empty locked list") } - // check if all objects are regular + err := db.lockInternal(locked, cnr, locker) + success = err == nil + return err +} + +func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { bucketKeysLocked := make([][]byte, len(locked)) for i := range locked { bucketKeysLocked[i] = objectKey(locked[i], make([]byte, objectKeySize)) @@ -83,7 +97,6 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. loop: for i := range bucketKeysLocked { - // decode list of already existing lockers exLockers, err = decodeList(bucketLockedContainer.Get(bucketKeysLocked[i])) if err != nil { return fmt.Errorf("decode list of object lockers: %w", err) @@ -95,14 +108,11 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } } - // update the list of lockers updLockers, err = encodeList(append(exLockers, keyLocker)) if err != nil { - // maybe continue for the best effort? return fmt.Errorf("encode list of object lockers: %w", err) } - // write updated list of lockers err = bucketLockedContainer.Put(bucketKeysLocked[i], updLockers) if err != nil { return fmt.Errorf("update list of object lockers: %w", err) @@ -116,6 +126,14 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. // FreeLockedBy unlocks all objects in DB which are locked by lockers. // Returns slice of unlocked object ID's or an error. func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("FreeLockedBy", time.Since(startedAt), success) + }() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -138,6 +156,7 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { }); err != nil { return nil, metaerr.Wrap(err) } + success = true return unlockedObjects, nil } @@ -280,6 +299,14 @@ func (i IsLockedRes) Locked() bool { // // Returns only non-logical errors related to underlying database. func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IsLocked", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IsLocked", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -292,9 +319,10 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e if db.mode.NoMetabase() { return res, ErrDegradedMode } - - return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err = metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.locked = objectLocked(tx, prm.addr.Container(), prm.addr.Object()) return nil })) + success = err == nil + return res, err } diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go new file mode 100644 index 000000000..120579e4c --- /dev/null +++ b/pkg/local_object_storage/metabase/metrics.go @@ -0,0 +1,20 @@ +package meta + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index dd1cdc900..28beca8f3 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -40,5 +40,6 @@ func (db *DB) SetMode(m mode.Mode) error { } db.mode = m + db.metrics.SetMode(m) return nil } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index b7cff68dc..7a24485b3 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" gio "io" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" @@ -58,6 +59,14 @@ var ( // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Put", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Put", trace.WithAttributes( attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), @@ -79,6 +88,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { return db.put(tx, prm.obj, prm.id, nil, currEpoch) }) if err == nil { + success = true storagelog.Write(db.log, storagelog.AddressField(objectCore.AddressOf(prm.obj)), storagelog.OpField("metabase PUT")) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 466476d4e..1f32d2cde 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -62,6 +63,14 @@ func (r SelectRes) AddressList() []oid.Address { // Select returns list of addresses of objects that match search filters. func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Select", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Select", trace.WithAttributes( attribute.String("container_id", prm.cnr.EncodeToString()), @@ -76,6 +85,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err } if blindlyProcess(prm.filters) { + success = true return res, nil } @@ -83,7 +93,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) - + success = err == nil return err })) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 84efa1d31..bc514933b 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -138,7 +138,7 @@ func (s *Shard) Init(ctx context.Context) error { } } - s.updateMetrics() + s.updateMetrics(ctx) s.gc = &gc{ gcCfg: &s.gcCfg, diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 2580173c2..b8d516871 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -270,7 +270,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { // iterate over metabase's objects with GC mark // (no more than s.rmBatchSize objects) - err := s.metaBase.IterateOverGarbage(iterPrm) + err := s.metaBase.IterateOverGarbage(ctx, iterPrm) if err != nil { s.log.Warn(logs.ShardIteratorOverMetabaseGraveyardFailed, zap.String("error", err.Error()), @@ -382,7 +382,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } - expired, err := s.getExpiredWithLinked(expired) + expired, err := s.getExpiredWithLinked(ctx, expired) if err != nil { s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) return @@ -414,9 +414,9 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } } -func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error) { +func (s *Shard) getExpiredWithLinked(ctx context.Context, source []oid.Address) ([]oid.Address, error) { result := make([]oid.Address, 0, len(source)) - parentToChildren, err := s.metaBase.GetChildren(source) + parentToChildren, err := s.metaBase.GetChildren(ctx, source) if err != nil { return nil, err } @@ -469,7 +469,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err = s.metaBase.IterateOverGraveyard(iterPrm) + err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) if err != nil { log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() @@ -560,7 +560,7 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo return ErrDegradedMode } - err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error { + err := s.metaBase.IterateExpired(ctx, epoch, func(expiredObject *meta.ExpiredObject) error { select { case <-ctx.Done(): return meta.ErrInterruptIterator @@ -628,7 +628,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // drop just processed expired tombstones // from graveyard - err = s.metaBase.DropGraves(tss) + err = s.metaBase.DropGraves(ctx, tss) if err != nil { s.log.Warn(logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) } diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index aaa1112cd..dd21745cc 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,8 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -65,7 +68,13 @@ func (r ListWithCursorRes) Cursor() *Cursor { } // List returns all objects physically stored in the Shard. -func (s *Shard) List() (res SelectRes, err error) { +func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.List", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -73,7 +82,7 @@ func (s *Shard) List() (res SelectRes, err error) { return SelectRes{}, ErrDegradedMode } - lst, err := s.metaBase.Containers() + lst, err := s.metaBase.Containers(ctx) if err != nil { return res, fmt.Errorf("can't list stored containers: %w", err) } @@ -86,7 +95,7 @@ func (s *Shard) List() (res SelectRes, err error) { sPrm.SetContainerID(lst[i]) sPrm.SetFilters(filters) - sRes, err := s.metaBase.Select(context.TODO(), sPrm) // consider making List in metabase + sRes, err := s.metaBase.Select(ctx, sPrm) // consider making List in metabase if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), @@ -101,12 +110,18 @@ func (s *Shard) List() (res SelectRes, err error) { return res, nil } -func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { +func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListContainersRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ListContainers", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListContainersRes{}, ErrDegradedMode } - containers, err := s.metaBase.Containers() + containers, err := s.metaBase.Containers(ctx) if err != nil { return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err) } @@ -122,7 +137,14 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "shard.ListWithCursor", + trace.WithAttributes( + attribute.Int64("count", int64(prm.count)), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListWithCursorRes{}, ErrDegradedMode } @@ -130,7 +152,7 @@ func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) var metaPrm meta.ListPrm metaPrm.SetCount(prm.count) metaPrm.SetCursor(prm.cursor) - res, err := s.metaBase.ListWithCursor(metaPrm) + res, err := s.metaBase.ListWithCursor(ctx, metaPrm) if err != nil { return ListWithCursorRes{}, fmt.Errorf("could not get list of objects: %w", err) } diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index bbce28430..63e7651c8 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -71,7 +71,7 @@ func testShardList(t *testing.T, sh *shard.Shard) { } require.NoError(t, errG.Wait()) - res, err := sh.List() + res, err := sh.List(context.Background()) require.NoError(t, err) for _, objID := range res.AddressList() { diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 1d2cab9f2..10c1acd40 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -370,7 +370,7 @@ const ( logical = "logic" ) -func (s *Shard) updateMetrics() { +func (s *Shard) updateMetrics(ctx context.Context) { if s.cfg.metricsWriter != nil && !s.GetMode().NoMetabase() { cc, err := s.metaBase.ObjectCounters() if err != nil { @@ -384,7 +384,7 @@ func (s *Shard) updateMetrics() { s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) - cnrList, err := s.metaBase.Containers() + cnrList, err := s.metaBase.Containers(ctx) if err != nil { s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) return diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 4a40f00ba..60e924755 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -34,7 +34,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { default: } - addrs, cursor, err = p.jobQueue.Select(cursor, p.batchSize) + addrs, cursor, err = p.jobQueue.Select(ctx, cursor, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { time.Sleep(time.Second) // finished whole cycle, sleep a bit diff --git a/pkg/services/policer/queue.go b/pkg/services/policer/queue.go index b8af44049..22012c835 100644 --- a/pkg/services/policer/queue.go +++ b/pkg/services/policer/queue.go @@ -1,6 +1,7 @@ package policer import ( + "context" "fmt" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -11,12 +12,12 @@ type jobQueue struct { localStorage *engine.StorageEngine } -func (q *jobQueue) Select(cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { +func (q *jobQueue) Select(ctx context.Context, cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { var prm engine.ListWithCursorPrm prm.WithCursor(cursor) prm.WithCount(count) - res, err := q.localStorage.ListWithCursor(prm) + res, err := q.localStorage.ListWithCursor(ctx, prm) if err != nil { return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err) } -- 2.45.2 From d5aaec1107bcc7437abe85418904df68e94414e4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 12:27:53 +0300 Subject: [PATCH 133/233] [#373] pilorama: Add metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/pilorama/boltdb.go | 145 +++++++++++++++++-- pkg/local_object_storage/pilorama/metrics.go | 20 +++ pkg/local_object_storage/pilorama/option.go | 7 + 3 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/metrics.go diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index e7bcb110b..1b8ee7f18 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -72,6 +72,7 @@ func NewBoltForest(opts ...Option) ForestStorage { maxBatchDelay: bbolt.DefaultMaxBatchDelay, maxBatchSize: bbolt.DefaultMaxBatchSize, openFile: os.OpenFile, + metrics: &noopMetrics{}, }, } @@ -101,6 +102,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { } t.mode = m + t.metrics.SetMode(m) return nil } func (t *boltForest) Open(readOnly bool) error { @@ -122,7 +124,11 @@ func (t *boltForest) Open(readOnly bool) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - + m := mode.ReadWrite + if readOnly { + m = mode.ReadOnly + } + t.metrics.SetMode(m) return nil } func (t *boltForest) Init() error { @@ -142,10 +148,14 @@ func (t *boltForest) Init() error { }) } func (t *boltForest) Close() error { + var err error if t.db != nil { - return t.db.Close() + err = t.db.Close() } - return nil + if err == nil { + t.metrics.Close() + } + return err } // TreeMove implements the Forest interface. @@ -225,6 +235,14 @@ func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID strin // TreeExists implements the Forest interface. func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeExists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -247,7 +265,7 @@ func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID strin exists = treeRoot != nil return nil }) - + success = err == nil return exists, metaerr.Wrap(err) } @@ -255,6 +273,14 @@ var syncHeightKey = []byte{'h'} // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeUpdateLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeUpdateLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -268,7 +294,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID binary.LittleEndian.PutUint64(rawHeight, height) buck := bucketName(cid, treeID) - return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(buck) if treeRoot == nil { return ErrTreeNotFound @@ -277,10 +303,20 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID b := treeRoot.Bucket(dataBucket) return b.Put(syncHeightKey, rawHeight) })) + success = err == nil + return err } // TreeLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -305,11 +341,20 @@ func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, tree } return nil }) + success = err == nil return height, metaerr.Wrap(err) } // TreeAddByPath implements the Forest interface. func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeAddByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeAddByPath", trace.WithAttributes( attribute.String("container_id", d.CID.EncodeToString()), @@ -323,6 +368,12 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID ) defer span.End() + res, err := t.addByPathInternal(d, attr, treeID, path, meta) + success = err == nil + return res, err +} + +func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID string, path []string, meta []KeyValue) ([]Move, error) { if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -417,6 +468,14 @@ func (t *boltForest) findSpareID(bTree *bbolt.Bucket) uint64 { // TreeApply implements the Forest interface. func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeApply", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApply", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -440,6 +499,7 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cnr, treeID)) if treeRoot == nil { + success = true return nil } @@ -448,16 +508,18 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var logKey [8]byte binary.BigEndian.PutUint64(logKey[:], m.Time) seen = b.Get(logKey[:]) != nil + success = true return nil }) if err != nil || seen { + success = err == nil return metaerr.Wrap(err) } } if t.db.MaxBatchSize == 1 { fullID := bucketName(cnr, treeID) - return metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -466,11 +528,15 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var lm Move return t.applyOperation(bLog, bTree, []*Move{m}, &lm) })) + success = err == nil + return err } ch := make(chan error, 1) t.addBatch(cnr, treeID, m, ch) - return metaerr.Wrap(<-ch) + err := <-ch + success = err == nil + return metaerr.Wrap(err) } func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { @@ -724,6 +790,14 @@ func (t *boltForest) isAncestor(b *bbolt.Bucket, parent, child Node) bool { // TreeGetByPath implements the Forest interface. func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetByPath", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -740,6 +814,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st } if len(path) == 0 { + success = true return nil, nil } @@ -752,7 +827,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st var nodes []Node - return nodes, metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { return ErrTreeNotFound @@ -790,10 +865,20 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st } return nil })) + success = err == nil + return nodes, err } // TreeGetMeta implements the forest interface. func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetMeta", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetMeta", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -828,12 +913,20 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri _, _, meta, _ := t.getState(b, stateKey(key, nodeID)) return m.FromBytes(meta) }) - + success = err == nil return m, parentID, metaerr.Wrap(err) } // TreeGetChildren implements the Forest interface. func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetChildren", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -869,12 +962,20 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID } return nil }) - + success = err == nil return children, metaerr.Wrap(err) } // TreeList implements the Forest interface. func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeList", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeList", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -910,12 +1011,20 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err if err != nil { return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) } - + success = true return ids, nil } // TreeGetOpLog implements the pilorama.Forest interface. func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetOpLog", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetOpLog", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -949,12 +1058,20 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str } return nil }) - + success = err == nil return lm, metaerr.Wrap(err) } // TreeDrop implements the pilorama.Forest interface. func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeDrop", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeDrop", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -972,7 +1089,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrReadOnlyMode } - return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { if treeID == "" { c := tx.Cursor() prefix := make([]byte, 32) @@ -991,6 +1108,8 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) } return err })) + success = err == nil + return err } func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go new file mode 100644 index 000000000..8b113eaa7 --- /dev/null +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -0,0 +1,20 @@ +package pilorama + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/pilorama/option.go b/pkg/local_object_storage/pilorama/option.go index 0dd5e63d4..d576d427f 100644 --- a/pkg/local_object_storage/pilorama/option.go +++ b/pkg/local_object_storage/pilorama/option.go @@ -15,6 +15,7 @@ type cfg struct { maxBatchDelay time.Duration maxBatchSize int openFile func(string, int, fs.FileMode) (*os.File, error) + metrics Metrics } func WithPath(path string) Option { @@ -52,3 +53,9 @@ func WithOpenFile(openFile func(string, int, fs.FileMode) (*os.File, error)) Opt c.openFile = openFile } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} -- 2.45.2 From d8ecc69d0091db029cb976456b8e923d332e1102 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 14:39:03 +0300 Subject: [PATCH 134/233] [#373] local storage: Pass parent ID This is required to add shard ID as metric label. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 4 ++++ .../blobstor/blobovniczatree/metrics.go | 3 +++ pkg/local_object_storage/blobstor/blobstor.go | 7 +++++++ pkg/local_object_storage/blobstor/common/storage.go | 1 + pkg/local_object_storage/blobstor/fstree/fstree.go | 4 ++++ pkg/local_object_storage/blobstor/fstree/metrics.go | 3 +++ pkg/local_object_storage/blobstor/memstore/control.go | 1 + pkg/local_object_storage/blobstor/metrics.go | 2 ++ pkg/local_object_storage/blobstor/teststore/teststore.go | 2 ++ pkg/local_object_storage/metabase/db.go | 5 +++++ pkg/local_object_storage/metabase/metrics.go | 3 +++ pkg/local_object_storage/pilorama/boltdb.go | 4 ++++ pkg/local_object_storage/pilorama/forest.go | 1 + pkg/local_object_storage/pilorama/interface.go | 1 + pkg/local_object_storage/pilorama/metrics.go | 3 +++ pkg/local_object_storage/shard/id.go | 3 +++ 16 files changed, 47 insertions(+) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d99e9aa89..1b0af342d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -256,3 +256,7 @@ func (b *Blobovniczas) SetCompressor(cc *compression.Config) { func (b *Blobovniczas) SetReportErrorFunc(f func(string, error)) { b.reportError = f } + +func (b *Blobovniczas) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index ffa3c8a8b..39fe34e28 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -9,6 +9,8 @@ import ( type Metrics interface { BlobovnicaMetrics(path string) blobovnicza.Metrics + SetParentID(parentID string) + SetMode(readOnly bool) Close() @@ -25,6 +27,7 @@ type noopMetrics struct{} func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { return &blobovnicza.NoopMetrics{} } +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index a422b2bce..6c6aed87f 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -72,6 +72,13 @@ func (b *BlobStor) SetLogger(l *logger.Logger) { b.log = l } +func (b *BlobStor) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) + for _, ss := range b.storage { + ss.Storage.SetParentID(parentID) + } +} + // WithStorages provides sub-blobstors. func WithStorages(st []SubStorage) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index c5d187f30..b808480e2 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -19,6 +19,7 @@ type Storage interface { // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. SetReportErrorFunc(f func(string, error)) + SetParentID(parentID string) Get(context.Context, GetPrm) (GetRes, error) GetRange(context.Context, GetRangePrm) (GetRangeRes, error) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 645d4e5f7..9ca5d4bd9 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -554,3 +554,7 @@ func (t *FSTree) SetCompressor(cc *compression.Config) { func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } + +func (t *FSTree) SetParentID(parentID string) { + t.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index 04386d9b8..ca6a54975 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -3,6 +3,8 @@ package fstree import "time" type Metrics interface { + SetParentID(parentID string) + SetMode(readOnly bool) Close() @@ -16,6 +18,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Iterate(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 4deb9f6e2..e6943626b 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -13,3 +13,4 @@ func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } func (s *memstoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f } +func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go index 4fd6f3e54..4a7b40092 100644 --- a/pkg/local_object_storage/blobstor/metrics.go +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -3,6 +3,7 @@ package blobstor import "time" type Metrics interface { + SetParentID(parentID string) SetMode(readOnly bool) Close() @@ -16,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index c7179eb45..2508d74f6 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -214,3 +214,5 @@ func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common. panic(fmt.Sprintf("unexpected storage call: Iterate(%+v)", req)) } } + +func (s *TestStore) SetParentID(string) {} diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index c54ed98d0..33f272117 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -294,6 +294,11 @@ func (db *DB) SetLogger(l *logger.Logger) { db.log = l } +// SetParentID sets parent ID to nested components. It is used after the shard ID was generated to use it in logs. +func (db *DB) SetParentID(parentID string) { + db.metrics.SetParentID(parentID) +} + // WithLogger returns option to set logger of DB. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go index 120579e4c..fc971bd81 100644 --- a/pkg/local_object_storage/metabase/metrics.go +++ b/pkg/local_object_storage/metabase/metrics.go @@ -7,6 +7,8 @@ import ( ) type Metrics interface { + SetParentID(parentID string) + SetMode(m mode.Mode) Close() @@ -15,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(mode.Mode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1b8ee7f18..a19561139 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -158,6 +158,10 @@ func (t *boltForest) Close() error { return err } +func (t *boltForest) SetParentID(id string) { + t.metrics.SetParentID(id) +} + // TreeMove implements the Forest interface. func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID string, m *Move) (*Move, error) { _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeMove", diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 7c5897ed0..76220c1db 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -119,6 +119,7 @@ func (f *memoryForest) SetMode(mode.Mode) error { func (f *memoryForest) Close() error { return nil } +func (f *memoryForest) SetParentID(string) {} // TreeGetByPath implements the Forest interface. func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index c8287c6d4..89c752627 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -61,6 +61,7 @@ type ForestStorage interface { Open(bool) error Close() error SetMode(m mode.Mode) error + SetParentID(id string) Forest } diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go index 8b113eaa7..543ad3e31 100644 --- a/pkg/local_object_storage/pilorama/metrics.go +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -7,6 +7,8 @@ import ( ) type Metrics interface { + SetParentID(id string) + SetMode(m mode.Mode) Close() @@ -15,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(mode.Mode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 992a86c01..4f6fd0651 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -55,6 +55,9 @@ func (s *Shard) UpdateID() (err error) { if s.hasWriteCache() { s.writeCache.SetLogger(s.log) } + s.metaBase.SetParentID(s.info.ID.String()) + s.blobStor.SetParentID(s.info.ID.String()) + s.pilorama.SetParentID(s.info.ID.String()) if len(id) != 0 { return nil -- 2.45.2 From 16a142cd0c403c51556e4e49a48fa1be02db9102 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 17:06:02 +0300 Subject: [PATCH 135/233] [#373] metrics: Add FSTree metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 21 +++-- pkg/local_object_storage/metrics/consts.go | 3 + pkg/local_object_storage/metrics/fstree.go | 61 ++++++++++++++ pkg/local_object_storage/shard/id.go | 4 +- pkg/metrics/consts.go | 15 ++++ pkg/metrics/fstree.go | 96 ++++++++++++++++++++++ pkg/metrics/mode.go | 36 ++++++++ pkg/metrics/node.go | 6 ++ 8 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 pkg/local_object_storage/metrics/consts.go create mode 100644 pkg/local_object_storage/metrics/fstree.go create mode 100644 pkg/metrics/consts.go create mode 100644 pkg/metrics/fstree.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c88b7224e..44da2e981 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -36,6 +36,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + lsmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -752,12 +753,22 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { }, }) case fstree.Type: + fstreeOpts := []fstree.Option{ + fstree.WithPath(sRead.path), + fstree.WithPerm(sRead.perm), + fstree.WithDepth(sRead.depth), + fstree.WithNoSync(sRead.noSync), + } + if c.metricsCollector != nil { + fstreeOpts = append(fstreeOpts, + fstree.WithMetrics( + lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ - Storage: fstree.New( - fstree.WithPath(sRead.path), - fstree.WithPerm(sRead.perm), - fstree.WithDepth(sRead.depth), - fstree.WithNoSync(sRead.noSync)), + Storage: fstree.New(fstreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return true }, diff --git a/pkg/local_object_storage/metrics/consts.go b/pkg/local_object_storage/metrics/consts.go new file mode 100644 index 000000000..519930710 --- /dev/null +++ b/pkg/local_object_storage/metrics/consts.go @@ -0,0 +1,3 @@ +package metrics + +const undefined = "undefined" diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go new file mode 100644 index 000000000..d3749d9bc --- /dev/null +++ b/pkg/local_object_storage/metrics/fstree.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { + return &fstreeMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type fstreeMetrics struct { + shardID string + path string + m metrics_impl.FSTreeMetrics +} + +func (m *fstreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *fstreeMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, m.path, readOnly) +} +func (m *fstreeMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) +} +func (m *fstreeMetrics) Delete(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) +} +func (m *fstreeMetrics) Exists(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) +} +func (m *fstreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Put", d, success) + if success { + m.m.AddPut(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) Get(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Get", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 4f6fd0651..e2ac423fd 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -57,7 +57,9 @@ func (s *Shard) UpdateID() (err error) { } s.metaBase.SetParentID(s.info.ID.String()) s.blobStor.SetParentID(s.info.ID.String()) - s.pilorama.SetParentID(s.info.ID.String()) + if s.pilorama != nil { + s.pilorama.SetParentID(s.info.ID.String()) + } if len(id) != 0 { return nil diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go new file mode 100644 index 000000000..d65ee51c1 --- /dev/null +++ b/pkg/metrics/consts.go @@ -0,0 +1,15 @@ +package metrics + +const ( + fstreeSubSystem = "fstree" + + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + + readWriteMode = "READ_WRITE" + readOnlyMode = "READ_ONLY" + closedMode = "CLOSED" +) diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go new file mode 100644 index 000000000..5bc43757a --- /dev/null +++ b/pkg/metrics/fstree.go @@ -0,0 +1,96 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type FSTreeMetrics interface { + SetMode(shardID, path string, readOnly bool) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) + AddGet(shardID, path string, size int) + AddPut(shardID, path string, size int) +} + +type fstreeMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newFSTreeMetrics() *fstreeMetrics { + return &fstreeMetrics{ + mode: newShardIDPathMode(fstreeSubSystem, "mode", "FSTree mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated FSTree request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to FSTree", + }, []string{shardIDLabel, pathLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from FSTree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + m.mode.SetMode(shardID, path, modeValue) +} + +func (m *fstreeMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *fstreeMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *fstreeMetrics) AddGet(shardID, path string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (m *fstreeMetrics) AddPut(shardID, path string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index 7ae3fd5f9..aa2246676 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -37,3 +37,39 @@ func (m *shardIDModeValue) Delete(shardID string) { wcShardID: shardID, }) } + +type shardIDPathModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDPathMode(subsystem, name, help string) *shardIDPathModeValue { + return &shardIDPathModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{shardIDLabel, pathLabel, modeLabel}), + } +} + +func (m *shardIDPathModeValue) SetMode(shardID, path string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + + m.modeValue.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + modeLabel: mode, + }).Set(1) +} + +func (m *shardIDPathModeValue) Delete(shardID, path string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 232c178d8..0b04b635b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -14,6 +14,7 @@ type NodeMetrics struct { objectService *objectServiceMetrics treeService *treeServiceMetrics epoch prometheus.Gauge + fstree *fstreeMetrics } func NewNodeMetrics() *NodeMetrics { @@ -41,6 +42,7 @@ func NewNodeMetrics() *NodeMetrics { replicator: replicator, treeService: treeService, epoch: epoch, + fstree: newFSTreeMetrics(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) Engine() EngineMetrics { func (m *NodeMetrics) State() StateMetrics { return m.state } + +func (m *NodeMetrics) FSTree() FSTreeMetrics { + return m.fstree +} -- 2.45.2 From 56f320dd85a5c065812becc9eeef15acecf0f371 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 18:04:23 +0300 Subject: [PATCH 136/233] [#373] metrics: Add blobstor metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 18 ++-- pkg/local_object_storage/metrics/blobstore.go | 65 ++++++++++++++ pkg/metrics/blobstore.go | 87 +++++++++++++++++++ pkg/metrics/consts.go | 14 +-- pkg/metrics/fstree.go | 6 +- pkg/metrics/mode.go | 8 ++ pkg/metrics/node.go | 42 ++++----- 7 files changed, 198 insertions(+), 42 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobstore.go create mode 100644 pkg/metrics/blobstore.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 44da2e981..bc73f8bc5 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -786,19 +786,23 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { piloramaOpts := c.getPiloramaOpts(shCfg) ss := c.getSubstorageOpts(shCfg) + blobstoreOpts := []blobstor.Option{ + blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), + blobstor.WithStorages(ss), + blobstor.WithLogger(c.log), + } + if c.metricsCollector != nil { + blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), - shard.WithBlobStorOptions( - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithStorages(ss), - - blobstor.WithLogger(c.log), - ), + shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions( meta.WithPath(shCfg.metaCfg.path), meta.WithPermissions(shCfg.metaCfg.perm), diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go new file mode 100644 index 000000000..48249e89c --- /dev/null +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +type blobstoreMetrics struct { + shardID string + m metrics_impl.BlobstoreMetrics +} + +func NewBlobstoreMetrics(m metrics_impl.BlobstoreMetrics) blobstor.Metrics { + return &blobstoreMetrics{ + shardID: undefined, + m: m, + } +} + +func (m *blobstoreMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobstoreMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, readOnly) +} + +func (m *blobstoreMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddPut(m.shardID, size) + } +} diff --git a/pkg/metrics/blobstore.go b/pkg/metrics/blobstore.go new file mode 100644 index 000000000..d9bb3f029 --- /dev/null +++ b/pkg/metrics/blobstore.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobstoreMetrics interface { + SetMode(shardID string, readOnly bool) + Close(shardID string) + + MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) + AddPut(shardID string, size int) + AddGet(shardID string, size int) +} + +type blobstoreMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newBlobstoreMetrics() *blobstoreMetrics { + return &blobstoreMetrics{ + mode: newShardIDMode(blobstoreSubSystem, "mode", "Blobstore mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobstore request process duration", + }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobstore", + }, []string{shardIDLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobstore", + }, []string{shardIDLabel}), + } +} + +func (m *blobstoreMetrics) SetMode(shardID string, readOnly bool) { + m.mode.SetMode(shardID, modeFromBool(readOnly)) +} + +func (m *blobstoreMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + +func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (m *blobstoreMetrics) AddPut(shardID string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} + +func (m *blobstoreMetrics) AddGet(shardID string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d65ee51c1..d06d15902 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,13 +1,15 @@ package metrics const ( - fstreeSubSystem = "fstree" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" - successLabel = "success" - shardIDLabel = "shardID" - modeLabel = "mode" - pathLabel = "path" - methodLabel = "method" + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "withStorageID" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go index 5bc43757a..4d4f0693b 100644 --- a/pkg/metrics/fstree.go +++ b/pkg/metrics/fstree.go @@ -49,11 +49,7 @@ func newFSTreeMetrics() *fstreeMetrics { } func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { - modeValue := readWriteMode - if readOnly { - modeValue = readOnlyMode - } - m.mode.SetMode(shardID, path, modeValue) + m.mode.SetMode(shardID, path, modeFromBool(readOnly)) } func (m *fstreeMetrics) Close(shardID, path string) { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index aa2246676..ad959e18e 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -73,3 +73,11 @@ func (m *shardIDPathModeValue) Delete(shardID, path string) { pathLabel: path, }) } + +func modeFromBool(readOnly bool) string { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + return modeValue +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0b04b635b..bf26aab61 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -15,34 +15,24 @@ type NodeMetrics struct { treeService *treeServiceMetrics epoch prometheus.Gauge fstree *fstreeMetrics + blobstore *blobstoreMetrics } func NewNodeMetrics() *NodeMetrics { - objectService := newObjectServiceMetrics() - - engine := newEngineMetrics() - - state := newStateMetrics() - - replicator := newReplicatorMetrics() - - treeService := newTreeServiceMetrics() - - epoch := metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: innerRingSubsystem, - Name: "epoch", - Help: "Current epoch as seen by inner-ring node.", - }) - return &NodeMetrics{ - objectService: objectService, - engine: engine, - state: state, - replicator: replicator, - treeService: treeService, - epoch: epoch, - fstree: newFSTreeMetrics(), + objectService: newObjectServiceMetrics(), + engine: newEngineMetrics(), + state: newStateMetrics(), + replicator: newReplicatorMetrics(), + treeService: newTreeServiceMetrics(), + epoch: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "epoch", + Help: "Current epoch as seen by inner-ring node.", + }), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), } } @@ -74,3 +64,7 @@ func (m *NodeMetrics) State() StateMetrics { func (m *NodeMetrics) FSTree() FSTreeMetrics { return m.fstree } + +func (m *NodeMetrics) Blobstore() BlobstoreMetrics { + return m.blobstore +} -- 2.45.2 From af608da952dbf9c531c7a31b008a81bfee55ef16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 12:14:32 +0300 Subject: [PATCH 137/233] [#373] metrics: Add blobovnizca metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 27 ++-- .../blobovnicza/control.go | 8 +- .../blobovnicza/metrics.go | 16 +- pkg/local_object_storage/blobovnicza/sizes.go | 4 +- .../blobstor/blobovniczatree/control.go | 2 +- .../blobstor/blobovniczatree/metrics.go | 8 +- .../metrics/blobovnizca.go | 98 ++++++++++++ pkg/metrics/blobovnizca.go | 141 ++++++++++++++++++ pkg/metrics/consts.go | 5 +- pkg/metrics/node.go | 10 +- 10 files changed, 287 insertions(+), 32 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobovnizca.go create mode 100644 pkg/metrics/blobovnizca.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index bc73f8bc5..0f2db8725 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -738,16 +738,25 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithLogger(c.log), + } - blobovniczatree.WithLogger(c.log)), + if c.metricsCollector != nil { + blobTreeOpts = append(blobTreeOpts, + blobovniczatree.WithMetrics( + lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index c776afe06..da4c870bd 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -36,7 +36,7 @@ func (b *Blobovnicza) Open() error { b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) if err == nil { - b.metrics.IncOpenCount() + b.metrics.IncOpenBlobovnizcaCount() } return err @@ -86,7 +86,7 @@ func (b *Blobovnicza) Init() error { sz := uint64(info.Size()) b.filled.Store(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) return err } @@ -98,8 +98,8 @@ func (b *Blobovnicza) Close() error { err := b.boltDB.Close() if err == nil { - b.metrics.DecOpenCount() - b.metrics.DecSize(b.filled.Load()) + b.metrics.DecOpenBlobovnizcaCount() + b.metrics.SubSize(b.filled.Load()) } return err } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 6127370bc..d511f90f1 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -1,16 +1,16 @@ package blobovnicza type Metrics interface { - IncOpenCount() - DecOpenCount() + IncOpenBlobovnizcaCount() + DecOpenBlobovnizcaCount() - IncSize(size uint64) - DecSize(size uint64) + AddSize(size uint64) + SubSize(size uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenCount() {} -func (m *NoopMetrics) DecOpenCount() {} -func (m *NoopMetrics) IncSize(uint64) {} -func (m *NoopMetrics) DecSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} +func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} +func (m *NoopMetrics) AddSize(uint64) {} +func (m *NoopMetrics) SubSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index bdbc77d18..7e10b728e 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,12 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.filled.Add(^(sz - 1)) - b.metrics.DecSize(sz) + b.metrics.SubSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0045e08db..bc3d7d60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -130,7 +130,7 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), blobovnicza.WithPath(path), - blobovnicza.WithMetrics(b.metrics.BlobovnicaMetrics(path)), + blobovnicza.WithMetrics(b.metrics.Blobovnizca()), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 39fe34e28..6c9b600c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -7,7 +7,7 @@ import ( ) type Metrics interface { - BlobovnicaMetrics(path string) blobovnicza.Metrics + Blobovnizca() blobovnicza.Metrics SetParentID(parentID string) @@ -24,9 +24,6 @@ type Metrics interface { type noopMetrics struct{} -func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { - return &blobovnicza.NoopMetrics{} -} func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} @@ -36,3 +33,6 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnizca.go new file mode 100644 index 000000000..a498d822e --- /dev/null +++ b/pkg/local_object_storage/metrics/blobovnizca.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { + return &blobovniczaTreeMetrics{ + path: path, + shardID: undefined, + m: m, + } +} + +type blobovniczaTreeMetrics struct { + shardID string + path string + m metrics_impl.BlobobvnizcaMetrics +} + +func (m *blobovniczaTreeMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovniczaMetrics{ + shardID: func() string { return m.shardID }, + path: m.path, + m: m.m, + } +} + +func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +} + +func (m *blobovniczaTreeMetrics) Close() { + m.m.CloseBlobobvnizcaTree(m.shardID, m.path) +} + +func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Exists", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "GetRange", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Get", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Iterate(d time.Duration, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobovniczaTreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddBlobobvnizcaTreePut(m.shardID, m.path, size) + } +} + +type blobovniczaMetrics struct { + m metrics_impl.BlobobvnizcaMetrics + shardID func() string + path string +} + +func (m *blobovniczaMetrics) AddSize(size uint64) { + m.m.AddTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) SubSize(size uint64) { + m.m.SubTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { + m.m.IncOpenBlobovnizcaCount(m.shardID(), m.path) +} + +func (m *blobovniczaMetrics) DecOpenBlobovnizcaCount() { + m.m.DecOpenBlobovnizcaCount(m.shardID(), m.path) +} diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnizca.go new file mode 100644 index 000000000..9dc3ed572 --- /dev/null +++ b/pkg/metrics/blobovnizca.go @@ -0,0 +1,141 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobobvnizcaMetrics interface { + SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + CloseBlobobvnizcaTree(shardID, path string) + BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) + AddBlobobvnizcaTreePut(shardID, path string, size int) + AddBlobobvnizcaTreeGet(shardID, path string, size int) + + AddTreeSize(shardID, path string, size uint64) + SubTreeSize(shardID, path string, size uint64) + + IncOpenBlobovnizcaCount(shardID, path string) + DecOpenBlobovnizcaCount(shardID, path string) +} + +type blobovnizca struct { + treeMode *shardIDPathModeValue + treeReqDuration *prometheus.HistogramVec + treePut *prometheus.CounterVec + treeGet *prometheus.CounterVec + treeSize *prometheus.GaugeVec + treeOpenCounter *prometheus.GaugeVec +} + +func newBlobovnizca() *blobovnizca { + return &blobovnizca{ + treeMode: newShardIDPathMode(blobovnizaTreeSubSystem, "mode", "Blobovnizca tree mode"), + + treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobovnizca tree request process duration", + }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), + treePut: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "size_bytes", + Help: "Blobovnizca tree size", + }, []string{shardIDLabel, pathLabel}), + treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "open_blobovnizca_count", + Help: "Count of opened blobovnizcas of Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (b *blobovnizca) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { + b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +} + +func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { + b.treeMode.SetMode(shardID, path, closedMode) + b.treeReqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treeGet.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treePut.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { + b.treeReqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (b *blobovnizca) AddBlobobvnizcaTreePut(shardID, path string, size int) { + b.treePut.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { + b.treeGet.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) SubTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Sub(float64(size)) +} + +func (b *blobovnizca) IncOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Inc() +} + +func (b *blobovnizca) DecOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Dec() +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d06d15902..d79e11696 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,8 +1,9 @@ package metrics const ( - fstreeSubSystem = "fstree" - blobstoreSubSystem = "blobstore" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" + blobovnizaTreeSubSystem = "blobovniza_tree" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index bf26aab61..babd6cdcf 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -16,6 +16,7 @@ type NodeMetrics struct { epoch prometheus.Gauge fstree *fstreeMetrics blobstore *blobstoreMetrics + blobobvnizca *blobovnizca } func NewNodeMetrics() *NodeMetrics { @@ -31,8 +32,9 @@ func NewNodeMetrics() *NodeMetrics { Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }), - fstree: newFSTreeMetrics(), - blobstore: newBlobstoreMetrics(), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), + blobobvnizca: newBlobovnizca(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) FSTree() FSTreeMetrics { func (m *NodeMetrics) Blobstore() BlobstoreMetrics { return m.blobstore } + +func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { + return m.blobobvnizca +} -- 2.45.2 From e89fa110c75230c341315c7ae76141a894388d29 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 12:45:31 +0300 Subject: [PATCH 138/233] [#373] metrics: Add metabase metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 28 +++++----- pkg/local_object_storage/metabase/db.go | 7 +++ pkg/local_object_storage/metrics/metabase.go | 39 ++++++++++++++ pkg/metrics/consts.go | 1 + pkg/metrics/metabase.go | 54 ++++++++++++++++++++ pkg/metrics/node.go | 6 +++ 6 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 pkg/local_object_storage/metrics/metabase.go create mode 100644 pkg/metrics/metabase.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 0f2db8725..1c8c49606 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -805,6 +805,21 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) } + mbOptions := []meta.Option{ + meta.WithPath(shCfg.metaCfg.path), + meta.WithPermissions(shCfg.metaCfg.perm), + meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), + meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), + meta.WithBoltDBOptions(&bbolt.Options{ + Timeout: 100 * time.Millisecond, + }), + meta.WithLogger(c.log), + meta.WithEpochState(c.cfgNetmap.state), + } + if c.metricsCollector != nil { + mbOptions = append(mbOptions, meta.WithMetrics(lsmetrics.NewMetabaseMetrics(shCfg.metaCfg.path, c.metricsCollector.MetabaseMetrics()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ @@ -812,18 +827,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), shard.WithBlobStorOptions(blobstoreOpts...), - shard.WithMetaBaseOptions( - meta.WithPath(shCfg.metaCfg.path), - meta.WithPermissions(shCfg.metaCfg.perm), - meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), - meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), - meta.WithBoltDBOptions(&bbolt.Options{ - Timeout: 100 * time.Millisecond, - }), - - meta.WithLogger(c.log), - meta.WithEpochState(c.cfgNetmap.state), - ), + shard.WithMetaBaseOptions(mbOptions...), shard.WithPiloramaOptions(piloramaOpts...), shard.WithWriteCache(shCfg.writecacheCfg.enabled), shard.WithWriteCacheOptions(writeCacheOpts...), diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 33f272117..e7de46e15 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -356,3 +356,10 @@ func WithEpochState(s EpochState) Option { c.epochState = s } } + +// WithMetrics returns option to specify metrics collector. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go new file mode 100644 index 000000000..d0fb31936 --- /dev/null +++ b/pkg/local_object_storage/metrics/metabase.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "time" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewMetabaseMetrics(path string, m metrics_impl.MetabaseMetrics) meta.Metrics { + return &metabaseMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type metabaseMetrics struct { + shardID string + path string + m metrics_impl.MetabaseMetrics +} + +func (m *metabaseMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *metabaseMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, m.path, mode.String()) +} + +func (m *metabaseMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *metabaseMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, method, d, success) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d79e11696..a9781abe1 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -4,6 +4,7 @@ const ( fstreeSubSystem = "fstree" blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" + metabaseSubSystem = "metabase" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/metabase.go b/pkg/metrics/metabase.go new file mode 100644 index 000000000..640c7f721 --- /dev/null +++ b/pkg/metrics/metabase.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type MetabaseMetrics interface { + SetMode(shardID, path string, mode string) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) +} + +func newMetabaseMetrics() *metabaseMetrics { + return &metabaseMetrics{ + mode: newShardIDPathMode(metabaseSubSystem, "mode", "Metabase mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: metabaseSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Metabase request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + } +} + +type metabaseMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *metabaseMetrics) SetMode(shardID, path string, mode string) { + m.mode.SetMode(shardID, path, mode) +} + +func (m *metabaseMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *metabaseMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index babd6cdcf..e4e98fe1f 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -17,6 +17,7 @@ type NodeMetrics struct { fstree *fstreeMetrics blobstore *blobstoreMetrics blobobvnizca *blobovnizca + metabase *metabaseMetrics } func NewNodeMetrics() *NodeMetrics { @@ -35,6 +36,7 @@ func NewNodeMetrics() *NodeMetrics { fstree: newFSTreeMetrics(), blobstore: newBlobstoreMetrics(), blobobvnizca: newBlobovnizca(), + metabase: newMetabaseMetrics(), } } @@ -74,3 +76,7 @@ func (m *NodeMetrics) Blobstore() BlobstoreMetrics { func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { return m.blobobvnizca } + +func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { + return m.metabase +} -- 2.45.2 From b5d9f4a2856d19db37251f655082b8f482444bab Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 16:19:47 +0300 Subject: [PATCH 139/233] [#373] metrics: Add pilorama metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++ pkg/local_object_storage/metrics/pilorama.go | 37 ++++++++++++++ pkg/metrics/consts.go | 1 + pkg/metrics/node.go | 6 +++ pkg/metrics/pilorama.go | 53 ++++++++++++++++++++ 5 files changed, 100 insertions(+) create mode 100644 pkg/local_object_storage/metrics/pilorama.go create mode 100644 pkg/metrics/pilorama.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 1c8c49606..2cb785d30 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -729,6 +729,9 @@ func (c *cfg) getPiloramaOpts(shCfg shardCfg) []pilorama.Option { pilorama.WithMaxBatchSize(prRead.maxBatchSize), pilorama.WithMaxBatchDelay(prRead.maxBatchDelay), ) + if c.metricsCollector != nil { + piloramaOpts = append(piloramaOpts, pilorama.WithMetrics(lsmetrics.NewPiloramaMetrics(c.metricsCollector.PiloramaMetrics()))) + } } return piloramaOpts } diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go new file mode 100644 index 000000000..21f027a6e --- /dev/null +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewPiloramaMetrics(m metrics_impl.PiloramaMetrics) pilorama.Metrics { + return &piloramaMetrics{ + shardID: undefined, + m: m, + } +} + +type piloramaMetrics struct { + shardID string + m metrics_impl.PiloramaMetrics +} + +func (m *piloramaMetrics) SetParentID(id string) { + m.shardID = id +} + +func (m *piloramaMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, mode) +} + +func (m *piloramaMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *piloramaMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.AddMethodDuration(m.shardID, method, d, success) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index a9781abe1..878e85a35 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -5,6 +5,7 @@ const ( blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" metabaseSubSystem = "metabase" + piloramaSubSystem = "pilorama" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index e4e98fe1f..3d3712450 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -18,6 +18,7 @@ type NodeMetrics struct { blobstore *blobstoreMetrics blobobvnizca *blobovnizca metabase *metabaseMetrics + pilorama *piloramaMetrics } func NewNodeMetrics() *NodeMetrics { @@ -37,6 +38,7 @@ func NewNodeMetrics() *NodeMetrics { blobstore: newBlobstoreMetrics(), blobobvnizca: newBlobovnizca(), metabase: newMetabaseMetrics(), + pilorama: newPiloramaMetrics(), } } @@ -80,3 +82,7 @@ func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { return m.metabase } + +func (m *NodeMetrics) PiloramaMetrics() PiloramaMetrics { + return m.pilorama +} diff --git a/pkg/metrics/pilorama.go b/pkg/metrics/pilorama.go new file mode 100644 index 000000000..41672a4b5 --- /dev/null +++ b/pkg/metrics/pilorama.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type PiloramaMetrics interface { + SetMode(shardID string, m mode.Mode) + Close(shardID string) + + AddMethodDuration(shardID string, method string, d time.Duration, success bool) +} + +func newPiloramaMetrics() *piloramaMetrics { + return &piloramaMetrics{ + mode: newShardIDMode(piloramaSubSystem, "mode", "Pilorama mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: piloramaSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Pilorama request process duration", + }, []string{shardIDLabel, successLabel, methodLabel}), + } +} + +type piloramaMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) +} + +func (m *piloramaMetrics) AddMethodDuration(shardID string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *piloramaMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} -- 2.45.2 From 03aa210145fe397cd5a53331adbc8df7aeb8ade4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 16 Jun 2023 10:13:22 +0300 Subject: [PATCH 140/233] [#373] metrics: Move labels to consts To unify label naming all lable keys and other consts are moved to one file. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/consts.go | 37 +++++++++++++++++---- pkg/metrics/engine.go | 35 +++++++++----------- pkg/metrics/gc.go | 40 +++++++++-------------- pkg/metrics/innerring.go | 13 ++------ pkg/metrics/mode.go | 10 +++--- pkg/metrics/morph.go | 23 ++++--------- pkg/metrics/morphcache.go | 14 +++----- pkg/metrics/node.go | 2 -- pkg/metrics/object.go | 20 +++--------- pkg/metrics/replicator.go | 2 -- pkg/metrics/state.go | 2 -- pkg/metrics/treeservice.go | 15 ++++----- pkg/metrics/writecache.go | 66 ++++++++++++++++---------------------- 13 files changed, 119 insertions(+), 160 deletions(-) diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index 878e85a35..ae3ac7839 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,20 +1,45 @@ package metrics const ( + namespace = "frostfs_node" + innerRingNamespace = "frostfs_ir" + fstreeSubSystem = "fstree" blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" metabaseSubSystem = "metabase" piloramaSubSystem = "pilorama" + engineSubsystem = "engine" + gcSubsystem = "garbage_collector" + innerRingSubsystem = "ir" + morphSubsystem = "morph" + morphCacheSubsystem = "morphcache" + objectSubsystem = "object" + replicatorSubsystem = "replicator" + stateSubsystem = "state" + treeServiceSubsystem = "treeservice" + writeCacheSubsystem = "writecache" - successLabel = "success" - shardIDLabel = "shardID" - modeLabel = "mode" - pathLabel = "path" - methodLabel = "method" - withStorageIDLabel = "withStorageID" + successLabel = "success" + shardIDLabel = "shard_id" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "with_storage_id" + statusLabel = "status" + objectTypeLabel = "object_type" + typeLabel = "type" + notificationTypeLabel = "notification_type" + invokeTypeLabel = "invoke_type" + contractLabel = "contract" + containerIDLabelKey = "cid" + storageLabel = "storage" + operationLabel = "operation" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" closedMode = "CLOSED" + + failedToDeleteStatus = "failed_to_delete" + deletedStatus = "deleted" ) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 395ab3356..23d799e28 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -35,23 +35,18 @@ type engineMetrics struct { writeCache *writeCacheMetrics } -const ( - engineSubsystem = "engine" - engineMethod = "method" -) - func newEngineMetrics() *engineMetrics { return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabel}), + errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabel}), methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: "request_duration_seconds", Help: "Duration of Engine requests", - }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + }, []string{methodLabel}), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabel, typeLabel}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), @@ -69,7 +64,7 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { m.methodDuration.With(prometheus.Labels{ - engineMethod: method, + methodLabel: method, }).Observe(d.Seconds()) } @@ -78,29 +73,29 @@ func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { } func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.With(prometheus.Labels{shardIDLabel: shardID}).Add(float64(size)) } func (m *engineMetrics) IncErrorCounter(shardID string) { - m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Inc() } func (m *engineMetrics) ClearErrorCounter(shardID string) { - m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Set(0) } func (m *engineMetrics) DeleteShardMetrics(shardID string) { - m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) - m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) - m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) + m.errorCounter.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.payloadSize.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.mode.Delete(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { m.objectCounter.With( prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, + shardIDLabel: shardID, + typeLabel: objectType, }, ).Add(float64(delta)) } @@ -108,8 +103,8 @@ func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { m.objectCounter.With( prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, + shardIDLabel: shardID, + typeLabel: objectType, }, ).Set(float64(v)) } diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index c0319562c..53bfef0e5 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -8,16 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - gcSubsystem = "garbage_collector" - gcShardID = "shard_id" - gcSuccess = "success" - gcStatus = "status" - gcDeleted = "deleted" - gcFailed = "failed_to_delete" - gcObjectType = "object_type" -) - type GCMetrics interface { AddRunDuration(shardID string, d time.Duration, success bool) AddDeletedCount(shardID string, deleted, failed uint64) @@ -39,60 +29,60 @@ func newGCMetrics() *gcMetrics { Subsystem: gcSubsystem, Name: "delete_duration_seconds", Help: "The total time of GC runs to delete objects from disk", - }, []string{gcShardID, gcSuccess}), + }, []string{shardIDLabel, successLabel}), deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "deleted_objects_total", Help: "Total count of objects GC deleted or failed to delete from disk", - }, []string{gcShardID, gcStatus}), + }, []string{shardIDLabel, statusLabel}), expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marking_duration_seconds", Help: "The total time of GC runs to mark expired objects as removed", - }, []string{gcShardID, gcSuccess, gcObjectType}), + }, []string{shardIDLabel, successLabel, objectTypeLabel}), inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marked_for_removal_objects_total", Help: "Total count of expired objects GC marked to remove", - }, []string{gcShardID, gcObjectType}), + }, []string{shardIDLabel, objectTypeLabel}), } } func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { m.runDuration.With(prometheus.Labels{ - gcShardID: shardID, - gcSuccess: strconv.FormatBool(success), + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), }).Add(d.Seconds()) } func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { m.deletedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcStatus: gcDeleted, + shardIDLabel: shardID, + statusLabel: deletedStatus, }).Add(float64(deleted)) m.deletedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcStatus: gcFailed, + shardIDLabel: shardID, + statusLabel: failedToDeleteStatus, }).Add(float64(failed)) } func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { m.expCollectDuration.With(prometheus.Labels{ - gcShardID: shardID, - gcSuccess: strconv.FormatBool(success), - gcObjectType: objectType, + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + objectTypeLabel: objectType, }).Add(d.Seconds()) } func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { m.inhumedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcObjectType: objectType, + shardIDLabel: shardID, + objectTypeLabel: objectType, }).Add(float64(count)) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 0aee068c5..d93b3c432 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -8,13 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - innerRingSubsystem = "ir" - innerRingLabelSuccess = "success" - innerRingLabelType = "type" - innerRingNamespace = "frostfs_ir" -) - // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { epoch prometheus.Gauge @@ -43,7 +36,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { Subsystem: innerRingSubsystem, Name: "event_duration_seconds", Help: "Duration of processing of inner-ring events", - }, []string{innerRingLabelType, innerRingLabelSuccess}) + }, []string{typeLabel, successLabel}) ) return &InnerRingServiceMetrics{ @@ -66,8 +59,8 @@ func (m *InnerRingServiceMetrics) SetHealth(s int32) { func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { m.eventDuration.With(prometheus.Labels{ - innerRingLabelType: typ, - innerRingLabelSuccess: strconv.FormatBool(success), + typeLabel: typ, + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index ad959e18e..312a6b33d 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -17,24 +17,24 @@ func newShardIDMode(subsystem, name, help string) *shardIDModeValue { Subsystem: subsystem, Name: name, Help: help, - }, []string{wcShardID, wcMode}), + }, []string{shardIDLabel, modeLabel}), } } func (m *shardIDModeValue) SetMode(shardID string, mode string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ - wcShardID: shardID, + shardIDLabel: shardID, }) m.modeValue.With(prometheus.Labels{ - wcShardID: shardID, - wcMode: mode, + shardIDLabel: shardID, + modeLabel: mode, }).Set(1) } func (m *shardIDModeValue) Delete(shardID string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ - wcShardID: shardID, + shardIDLabel: shardID, }) } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go index cd5deb5e6..5215c674b 100644 --- a/pkg/metrics/morph.go +++ b/pkg/metrics/morph.go @@ -9,15 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - morphSubsystem = "morph" - morphNotificationTypeLabel = "notification_type" - morphInvokeTypeLabel = "invoke_type" - morphContractLabel = "contract" - morphMethodLabel = "method" - morphSuccessLabel = "success" -) - type morphClientMetrics struct { switchCount prometheus.Counter lastBlock prometheus.Gauge @@ -44,13 +35,13 @@ func NewMorphClientMetrics() morphmetrics.Register { Subsystem: morphSubsystem, Name: "notifications_total", Help: "Number of notifications received by notification type", - }, []string{morphNotificationTypeLabel}), + }, []string{notificationTypeLabel}), invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: morphSubsystem, Name: "invoke_duration_seconds", Help: "Cummulative duration of contract invocations", - }, []string{morphInvokeTypeLabel, morphContractLabel, morphMethodLabel, morphSuccessLabel}), + }, []string{invokeTypeLabel, contractLabel, methodLabel, successLabel}), } } @@ -65,7 +56,7 @@ func (m *morphClientMetrics) SetLastBlock(index uint32) { func (m *morphClientMetrics) IncNotificationCount(typ string) { m.notificationCount.With( prometheus.Labels{ - morphNotificationTypeLabel: typ, + notificationTypeLabel: typ, }, ).Inc() } @@ -73,10 +64,10 @@ func (m *morphClientMetrics) IncNotificationCount(typ string) { func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { m.invokeDuration.With( prometheus.Labels{ - morphInvokeTypeLabel: typ, - morphContractLabel: contract, - morphMethodLabel: method, - morphSuccessLabel: strconv.FormatBool(success), + invokeTypeLabel: typ, + contractLabel: contract, + methodLabel: method, + successLabel: strconv.FormatBool(success), }, ).Observe(d.Seconds()) } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 3f215b5bf..a4dbbccfc 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -8,12 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - mcSubsystem = "morphcache" - mcSuccess = "success" - mcMethod = "method" -) - type MorphCacheMetrics interface { AddMethodDuration(method string, success bool, d time.Duration) } @@ -32,18 +26,18 @@ func newMorphCacheMetrics(ns string) *morphCacheMetrics { return &morphCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: ns, - Subsystem: mcSubsystem, + Subsystem: morphCacheSubsystem, Name: "request_duration_seconds", Help: "Morph cache request process duration", - }, []string{mcSuccess, mcMethod}), + }, []string{successLabel, methodLabel}), } } func (m *morphCacheMetrics) AddMethodDuration(method string, success bool, d time.Duration) { m.methodDuration.With( prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - mcMethod: method, + successLabel: strconv.FormatBool(success), + methodLabel: method, }, ).Observe(d.Seconds()) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 3d3712450..45d50b5b0 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const namespace = "frostfs_node" - type NodeMetrics struct { engine *engineMetrics state *stateMetrics diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 6e1b3a5c4..0ba994ed3 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -8,8 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const objectSubsystem = "object" - type ObjectServiceMetrics interface { AddRequestDuration(method string, d time.Duration, success bool) AddPayloadSize(method string, size int) @@ -20,14 +18,6 @@ type objectServiceMetrics struct { payloadCounter *prometheus.CounterVec } -const ( - shardIDLabelKey = "shard" - counterTypeLabelKey = "type" - containerIDLabelKey = "cid" - methodLabelKey = "method" - successLabelKey = "success" -) - func newObjectServiceMetrics() *objectServiceMetrics { return &objectServiceMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ @@ -35,25 +25,25 @@ func newObjectServiceMetrics() *objectServiceMetrics { Subsystem: objectSubsystem, Name: "request_duration_seconds", Help: "Object Service request process duration", - }, []string{methodLabelKey, successLabelKey}), + }, []string{methodLabel, successLabel}), payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: "request_payload_bytes", Help: "Object Service request payload", - }, []string{methodLabelKey}), + }, []string{methodLabel}), } } func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { m.methodDuration.With(prometheus.Labels{ - methodLabelKey: method, - successLabelKey: strconv.FormatBool(success), + methodLabel: method, + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { m.payloadCounter.With(prometheus.Labels{ - methodLabelKey: method, + methodLabel: method, }).Add(float64(size)) } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index f16c296c6..a1519ac95 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const replicatorSubsystem = "replicator" - //TODO type ReplicatorMetrics interface { diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 76a8d8ac9..243f648e5 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const stateSubsystem = "state" - type StateMetrics interface { SetHealth(s int32) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index ae24c41b2..6702aa83c 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -8,8 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const treeServiceLabelSuccess = "success" - type TreeMetricsRegister interface { AddReplicateTaskDuration(time.Duration, bool) AddReplicateWaitDuration(time.Duration, bool) @@ -25,43 +23,42 @@ type treeServiceMetrics struct { var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) func newTreeServiceMetrics() *treeServiceMetrics { - const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ replicateTaskDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_task_duration_seconds", Help: "Duration of individual replication tasks executed as part of replication loops", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), replicateWaitDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_wait_duration_seconds", Help: "Duration of overall waiting time for replication loops", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), syncOpDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "sync_duration_seconds", Help: "Duration of synchronization operations", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), } } func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { m.replicateTaskDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { m.replicateWaitDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { m.syncOpDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index a9c1b33a7..7e6083a49 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -8,16 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - wcSubsystem = "writecache" - wcShardID = "shard_id" - wcSuccess = "success" - wcStorage = "storage" - wcMode = "mode" - wcMethod = "method" - wcOperation = "operation" -) - type WriteCacheMetrics interface { AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) @@ -48,58 +38,58 @@ func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: "request_duration_seconds", Help: "Writecache request process duration", - }, []string{wcShardID, wcSuccess, wcStorage, wcMethod}), + }, []string{shardIDLabel, successLabel, storageLabel, methodLabel}), operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: "operations_total", Help: "The number of writecache operations processed", - }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), - actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), - estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - mode: newShardIDMode(wcSubsystem, "mode_info", "Writecache mode value"), + }, []string{shardIDLabel, storageLabel, successLabel, operationLabel}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{shardIDLabel, storageLabel}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{shardIDLabel, storageLabel}), + mode: newShardIDMode(writeCacheSubsystem, "mode_info", "Writecache mode value"), } } func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { m.methodDuration.With( prometheus.Labels{ - wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), - wcStorage: storageType, - wcMethod: method, + shardIDLabel: shardID, + successLabel: fmt.Sprintf("%v", success), + storageLabel: storageType, + methodLabel: method, }, ).Observe(d.Seconds()) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Inc() } func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Dec() } func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Set(float64(count)) } func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { m.estimatedSize.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Set(float64(size)) } @@ -109,25 +99,25 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { m.operationCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, - wcOperation: operation, - wcSuccess: success.String(), + shardIDLabel: shardID, + storageLabel: storageType, + operationLabel: operation, + successLabel: success.String(), }).Inc() } func (m *writeCacheMetrics) Close(shardID string) { m.mode.Delete(shardID) - m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.estimatedSize.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.methodDuration.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.actualCount.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) } func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: name, Help: help, }, labels) -- 2.45.2 From 4d48377cec9f78046e1f2bb37def07be3e98e890 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 21 Jun 2023 16:42:58 +0300 Subject: [PATCH 141/233] [#459] blobovniczatree: Fix get error Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/get.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 5dafedd1c..e88310f8a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -57,6 +57,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G success = true size = len(res.RawData) } + return res, err } activeCache := make(map[string]struct{}) -- 2.45.2 From 785d81a68ae347aa6d991132d3da60b71bed479d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Jun 2023 13:52:26 +0300 Subject: [PATCH 142/233] [#460] services/object: Reduce `distibutedTarget` memory footprint Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 11 ++--------- pkg/services/object/put/streamer.go | 15 ++++++++++----- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index b24218621..51678fa7b 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -25,8 +25,6 @@ type preparedObjectTarget interface { type distributedTarget struct { traversal traversal - remotePool, localPool util.WorkerPool - obj *objectSDK.Object objMeta object.ContentMeta @@ -34,7 +32,7 @@ type distributedTarget struct { nodeTargetInitializer func(nodeDesc) preparedObjectTarget - isLocalKey func([]byte) bool + getWorkerPool func([]byte) (util.WorkerPool, bool) relay func(context.Context, nodeDesc) error @@ -223,13 +221,8 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla wg.Add(1) addr := addrs[i] - isLocal := t.isLocalKey(addr.PublicKey()) - - workerPool := t.remotePool - if isLocal { - workerPool = t.localPool - } + workerPool, isLocal := t.getWorkerPool(addr.PublicKey()) if err := workerPool.Submit(func() { defer wg.Done() diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 6d0d8062e..8be9c75a0 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" @@ -219,9 +220,8 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { extraBroadcastEnabled: withBroadcast, }, - payload: getPayload(), - remotePool: p.remotePool, - localPool: p.localPool, + payload: getPayload(), + getWorkerPool: p.getWorkerPool, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return &localTarget{ @@ -242,8 +242,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { relay: relay, fmt: p.fmtValidator, log: p.log, - - isLocalKey: p.netmapKeys.IsLocalKey, } } @@ -280,3 +278,10 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { id: ids.SelfID, }, nil } + +func (p *Streamer) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { + if p.netmapKeys.IsLocalKey(pub) { + return p.localPool, true + } + return p.remotePool, false +} -- 2.45.2 From 167a67f0b846cdcfb5405f5bc6f3d13057e3161f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 17:48:50 +0300 Subject: [PATCH 143/233] [#460] services/util: Remove `HandleUnaryRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/accounting.go | 6 +- cmd/frostfs-node/container.go | 19 ++-- cmd/frostfs-node/netmap.go | 20 ++-- cmd/frostfs-node/session.go | 5 +- pkg/services/accounting/executor.go | 10 +- pkg/services/accounting/response.go | 37 ------- pkg/services/container/executor.go | 14 ++- pkg/services/container/response.go | 115 -------------------- pkg/services/netmap/executor.go | 11 +- pkg/services/netmap/response.go | 63 ----------- pkg/services/object/response.go | 27 ++--- pkg/services/session/executor.go | 11 +- pkg/services/session/response.go | 37 ------- pkg/services/util/response/client_stream.go | 6 +- pkg/services/util/response/server_stream.go | 6 +- pkg/services/util/response/service.go | 7 +- pkg/services/util/response/unary.go | 21 ---- 17 files changed, 76 insertions(+), 339 deletions(-) delete mode 100644 pkg/services/accounting/response.go delete mode 100644 pkg/services/container/response.go delete mode 100644 pkg/services/netmap/response.go delete mode 100644 pkg/services/session/response.go delete mode 100644 pkg/services/util/response/unary.go diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 6a35f37d0..d04f34ff1 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -21,10 +21,8 @@ func initAccountingService(ctx context.Context, c *cfg) { server := accountingTransportGRPC.New( accountingService.NewSignService( &c.key.PrivateKey, - accountingService.NewResponseService( - accountingService.NewExecutionService( - accounting.NewExecutor(balanceMorphWrapper), - ), + accountingService.NewExecutionService( + accounting.NewExecutor(balanceMorphWrapper), c.respSvc, ), ), diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d5a5afce1..9a6cfd02b 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -83,16 +83,13 @@ func initContainerService(ctx context.Context, c *cfg) { server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, - containerService.NewResponseService( - &usedSpaceService{ - Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt)), - loadWriterProvider: loadRouter, - loadPlacementBuilder: loadPlacementBuilder, - routeBuilder: routeBuilder, - cfg: c, - }, - c.respSvc, - ), + &usedSpaceService{ + Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + loadWriterProvider: loadRouter, + loadPlacementBuilder: loadPlacementBuilder, + routeBuilder: routeBuilder, + cfg: c, + }, ), ) @@ -575,6 +572,8 @@ func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *container resp := new(containerV2.AnnounceUsedSpaceResponse) resp.SetBody(respBody) + c.cfg.respSvc.SetMeta(resp) + return resp, nil } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index f1ea8b40e..b7b09a00b 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -148,17 +148,15 @@ func initNetmapService(ctx context.Context, c *cfg) { server := netmapTransportGRPC.New( netmapService.NewSignService( &c.key.PrivateKey, - netmapService.NewResponseService( - netmapService.NewExecutionService( - c, - c.apiVersion, - &netInfo{ - netState: c.cfgNetmap.state, - magic: c.cfgMorph.client, - morphClientNetMap: c.cfgNetmap.wrapper, - msPerBlockRdr: c.cfgMorph.client.MsPerBlock, - }, - ), + netmapService.NewExecutionService( + c, + c.apiVersion, + &netInfo{ + netState: c.cfgNetmap.state, + magic: c.cfgMorph.client, + morphClientNetMap: c.cfgNetmap.wrapper, + msPerBlockRdr: c.cfgMorph.client.MsPerBlock, + }, c.respSvc, ), ), diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 95e3b8205..f9c1811a1 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -53,10 +53,7 @@ func initSessionService(c *cfg) { server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, - sessionSvc.NewResponseService( - sessionSvc.NewExecutionService(c.privateTokenStore, c.log), - c.respSvc, - ), + sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), ), ) diff --git a/pkg/services/accounting/executor.go b/pkg/services/accounting/executor.go index 402845957..b0722cf8a 100644 --- a/pkg/services/accounting/executor.go +++ b/pkg/services/accounting/executor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -12,13 +13,15 @@ type ServiceExecutor interface { } type executorSvc struct { - exec ServiceExecutor + exec ServiceExecutor + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Accounting Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -31,5 +34,6 @@ func (s *executorSvc) Balance(ctx context.Context, req *accounting.BalanceReques resp := new(accounting.BalanceResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/accounting/response.go b/pkg/services/accounting/response.go deleted file mode 100644 index a78ac6fd6..000000000 --- a/pkg/services/accounting/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package accounting - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns accounting service instance that passes internal service -// call to response service. -func NewResponseService(accSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: accSvc, - } -} - -func (s *responseService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil -} diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index b4705d258..d4ae11d62 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -21,12 +22,15 @@ type executorSvc struct { Server exec ServiceExecutor + + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Container Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -44,6 +48,7 @@ func (s *executorSvc) Put(ctx context.Context, req *container.PutRequest) (*cont resp := new(container.PutResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -61,6 +66,7 @@ func (s *executorSvc) Delete(ctx context.Context, req *container.DeleteRequest) resp := new(container.DeleteResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -73,6 +79,7 @@ func (s *executorSvc) Get(ctx context.Context, req *container.GetRequest) (*cont resp := new(container.GetResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -85,6 +92,7 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co resp := new(container.ListResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -102,6 +110,7 @@ func (s *executorSvc) SetExtendedACL(ctx context.Context, req *container.SetExte resp := new(container.SetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -114,5 +123,6 @@ func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExte resp := new(container.GetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/container/response.go b/pkg/services/container/response.go deleted file mode 100644 index 138974537..000000000 --- a/pkg/services/container/response.go +++ /dev/null @@ -1,115 +0,0 @@ -package container - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns container service instance that passes internal service -// call to response service. -func NewResponseService(cnrSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: cnrSvc, - } -} - -func (s *responseService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.PutResponse), nil -} - -func (s *responseService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.DeleteResponse), nil -} - -func (s *responseService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetResponse), nil -} - -func (s *responseService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.ListResponse), nil -} - -func (s *responseService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.SetExtendedACLResponse), nil -} - -func (s *responseService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetExtendedACLResponse), nil -} - -func (s *responseService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.AnnounceUsedSpaceResponse), nil -} diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index d77c69d4d..d1e7a949e 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" versionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -18,6 +19,8 @@ type executorSvc struct { state NodeState netInfo NetworkInfo + + respSvc *response.Service } // NodeState encapsulates information @@ -42,8 +45,8 @@ type NetworkInfo interface { Dump(versionsdk.Version) (*netmapSDK.NetworkInfo, error) } -func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) Server { - if s == nil || netInfo == nil || !version.IsValid(v) { +func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { + if s == nil || netInfo == nil || !version.IsValid(v) || respSvc == nil { // this should never happen, otherwise it programmers bug panic("can't create netmap execution service") } @@ -51,6 +54,7 @@ func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) res := &executorSvc{ state: s, netInfo: netInfo, + respSvc: respSvc, } v.WriteToV2(&res.version) @@ -96,6 +100,7 @@ func (s *executorSvc) LocalNodeInfo( resp := new(netmap.LocalNodeInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -126,6 +131,7 @@ func (s *executorSvc) NetworkInfo( resp := new(netmap.NetworkInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -143,5 +149,6 @@ func (s *executorSvc) Snapshot(_ context.Context, _ *netmap.SnapshotRequest) (*n resp := new(netmap.SnapshotResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/netmap/response.go b/pkg/services/netmap/response.go deleted file mode 100644 index 8b035e461..000000000 --- a/pkg/services/netmap/response.go +++ /dev/null @@ -1,63 +0,0 @@ -package netmap - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns netmap service instance that passes internal service -// call to response service. -func NewResponseService(nmSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: nmSvc, - } -} - -func (s *responseService) LocalNodeInfo(ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.LocalNodeInfoResponse), nil -} - -func (s *responseService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.NetworkInfoResponse), nil -} - -func (s *responseService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.SnapshotResponse), nil -} diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index def934ea6..4eef1dfb9 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -91,16 +91,13 @@ func (s *ResponseService) Put() (PutObjectStream, error) { } func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - ) + resp, err := s.svc.Head(ctx, req) if err != nil { return nil, err } - return resp.(*object.HeadResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } func (s *searchStreamResponser) Send(resp *object.SearchResponse) error { @@ -117,16 +114,13 @@ func (s *ResponseService) Search(req *object.SearchRequest, stream SearchStream) } func (s *ResponseService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - ) + resp, err := s.svc.Delete(ctx, req) if err != nil { return nil, err } - return resp.(*object.DeleteResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } func (s *getRangeStreamResponser) Send(resp *object.GetRangeResponse) error { @@ -143,14 +137,11 @@ func (s *ResponseService) GetRange(req *object.GetRangeRequest, stream GetObject } func (s *ResponseService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - ) + resp, err := s.svc.GetRangeHash(ctx, req) if err != nil { return nil, err } - return resp.(*object.GetRangeHashResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 5ad1d6518..76c220fab 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -17,14 +18,17 @@ type ServiceExecutor interface { type executorSvc struct { exec ServiceExecutor + respSvc *response.Service + log *logger.Logger } // NewExecutionService wraps ServiceExecutor and returns Session Service interface. -func NewExecutionService(exec ServiceExecutor, l *logger.Logger) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *logger.Logger) Server { return &executorSvc{ - exec: exec, - log: l, + exec: exec, + log: l, + respSvc: respSvc, } } @@ -42,5 +46,6 @@ func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (* resp := new(session.CreateResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/session/response.go b/pkg/services/session/response.go deleted file mode 100644 index cbf93fb1f..000000000 --- a/pkg/services/session/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package session - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns session service instance that passes internal service -// call to response service. -func NewResponseService(ssSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: ssSvc, - } -} - -func (s *responseService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*session.CreateResponse), nil -} diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go index b541c73db..962f2bc26 100644 --- a/pkg/services/util/response/client_stream.go +++ b/pkg/services/util/response/client_stream.go @@ -10,7 +10,7 @@ import ( // ClientMessageStreamer represents client-side message streamer // that sets meta values to the response. type ClientMessageStreamer struct { - cfg *cfg + srv *Service send util.RequestMessageWriter @@ -33,7 +33,7 @@ func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.Response return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) } - setMeta(resp, s.cfg) + s.srv.SetMeta(resp) return resp, nil } @@ -41,7 +41,7 @@ func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.Response // CreateRequestStreamer wraps stream methods and returns ClientMessageStreamer instance. func (s *Service) CreateRequestStreamer(sender util.RequestMessageWriter, closer util.ClientStreamCloser) *ClientMessageStreamer { return &ClientMessageStreamer{ - cfg: s.cfg, + srv: s, send: sender, close: closer, } diff --git a/pkg/services/util/response/server_stream.go b/pkg/services/util/response/server_stream.go index 8a19fc4e7..df1c00177 100644 --- a/pkg/services/util/response/server_stream.go +++ b/pkg/services/util/response/server_stream.go @@ -9,7 +9,7 @@ import ( // ServerMessageStreamer represents server-side message streamer // that sets meta values to all response messages. type ServerMessageStreamer struct { - cfg *cfg + srv *Service recv util.ResponseMessageReader } @@ -22,7 +22,7 @@ func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { return nil, fmt.Errorf("could not receive response message for signing: %w", err) } - setMeta(m, s.cfg) + s.srv.SetMeta(m) return m, nil } @@ -30,7 +30,7 @@ func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { // HandleServerStreamRequest builds internal streamer via handlers, wraps it to ServerMessageStreamer and returns the result. func (s *Service) HandleServerStreamRequest(respWriter util.ResponseMessageWriter) util.ResponseMessageWriter { return func(resp util.ResponseMessage) error { - setMeta(resp, s.cfg) + s.SetMeta(resp) return respWriter(resp) } diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index 87cc8383d..a63d5343b 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -44,11 +44,12 @@ func NewService(opts ...Option) *Service { } } -func setMeta(resp util.ResponseMessage, cfg *cfg) { +// SetMeta sets adds meta-header to resp. +func (s *Service) SetMeta(resp util.ResponseMessage) { meta := new(session.ResponseMetaHeader) - meta.SetVersion(&cfg.version) + meta.SetVersion(&s.cfg.version) meta.SetTTL(1) // FIXME: #1160 TTL must be calculated - meta.SetEpoch(cfg.state.CurrentEpoch()) + meta.SetEpoch(s.cfg.state.CurrentEpoch()) if origin := resp.GetMetaHeader(); origin != nil { // FIXME: #1160 what if origin is set by local server? diff --git a/pkg/services/util/response/unary.go b/pkg/services/util/response/unary.go deleted file mode 100644 index 29cb95314..000000000 --- a/pkg/services/util/response/unary.go +++ /dev/null @@ -1,21 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// HandleUnaryRequest call passes request to handler, sets response meta header values and returns it. -func (s *Service) HandleUnaryRequest(ctx context.Context, req any, handler util.UnaryHandler) (util.ResponseMessage, error) { - // process request - resp, err := handler(ctx, req) - if err != nil { - return nil, fmt.Errorf("could not handle request: %w", err) - } - - setMeta(resp, s.cfg) - - return resp, nil -} -- 2.45.2 From a64dc9ad7083e2021b202aefffe02d48ce221b5d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 17:55:03 +0300 Subject: [PATCH 144/233] [#460] services/util: Remove `HandleServerStreamRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/response.go | 39 ++++++++++----------- pkg/services/util/response/server_stream.go | 37 ------------------- 2 files changed, 18 insertions(+), 58 deletions(-) delete mode 100644 pkg/services/util/response/server_stream.go diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 4eef1dfb9..0bec72163 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -16,21 +16,21 @@ type ResponseService struct { } type searchStreamResponser struct { - util.ServerStream + SearchStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getStreamResponser struct { - util.ServerStream + GetObjectStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getRangeStreamResponser struct { - util.ServerStream + GetObjectRangeStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type putStreamResponser struct { @@ -47,15 +47,14 @@ func NewResponseService(objSvc ServiceServer, respSvc *response.Service) *Respon } func (s *getStreamResponser) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectStream.Send(resp) } func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) error { return s.svc.Get(req, &getStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }), + GetObjectStream: stream, + respSvc: s.respSvc, }) } @@ -101,15 +100,14 @@ func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*o } func (s *searchStreamResponser) Send(resp *object.SearchResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.SearchStream.Send(resp) } func (s *ResponseService) Search(req *object.SearchRequest, stream SearchStream) error { return s.svc.Search(req, &searchStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }), + SearchStream: stream, + respSvc: s.respSvc, }) } @@ -124,15 +122,14 @@ func (s *ResponseService) Delete(ctx context.Context, req *object.DeleteRequest) } func (s *getRangeStreamResponser) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectRangeStream.Send(resp) } func (s *ResponseService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { return s.svc.GetRange(req, &getRangeStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }), + GetObjectRangeStream: stream, + respSvc: s.respSvc, }) } diff --git a/pkg/services/util/response/server_stream.go b/pkg/services/util/response/server_stream.go deleted file mode 100644 index df1c00177..000000000 --- a/pkg/services/util/response/server_stream.go +++ /dev/null @@ -1,37 +0,0 @@ -package response - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ServerMessageStreamer represents server-side message streamer -// that sets meta values to all response messages. -type ServerMessageStreamer struct { - srv *Service - - recv util.ResponseMessageReader -} - -// Recv calls Recv method of internal streamer, sets response meta -// values and returns the response. -func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { - m, err := s.recv() - if err != nil { - return nil, fmt.Errorf("could not receive response message for signing: %w", err) - } - - s.srv.SetMeta(m) - - return m, nil -} - -// HandleServerStreamRequest builds internal streamer via handlers, wraps it to ServerMessageStreamer and returns the result. -func (s *Service) HandleServerStreamRequest(respWriter util.ResponseMessageWriter) util.ResponseMessageWriter { - return func(resp util.ResponseMessage) error { - s.SetMeta(resp) - - return respWriter(resp) - } -} -- 2.45.2 From 40eae221099b5c4bd872ab4331d22cae3a92f364 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:01:51 +0300 Subject: [PATCH 145/233] [#460] services/util: Remove `CreateRequestStreamer` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/response.go | 24 +++++------ pkg/services/util/response/client_stream.go | 48 --------------------- 2 files changed, 11 insertions(+), 61 deletions(-) delete mode 100644 pkg/services/util/response/client_stream.go diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 0bec72163..81d6aaaaf 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) @@ -34,7 +33,8 @@ type getRangeStreamResponser struct { } type putStreamResponser struct { - stream *response.ClientMessageStreamer + stream PutObjectStream + respSvc *response.Service } // NewResponseService returns object service instance that passes internal service @@ -59,16 +59,20 @@ func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) er } func (s *putStreamResponser) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + if err := s.stream.Send(ctx, req); err != nil { + return fmt.Errorf("could not send the request: %w", err) + } + return nil } func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { r, err := s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("(%T) could not receive response: %w", s, err) + return nil, fmt.Errorf("could not close stream and receive response: %w", err) } - return r.(*object.PutResponse), nil + s.respSvc.SetMeta(r) + return r, nil } func (s *ResponseService) Put() (PutObjectStream, error) { @@ -78,14 +82,8 @@ func (s *ResponseService) Put() (PutObjectStream, error) { } return &putStreamResponser{ - stream: s.respSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - ), + stream: stream, + respSvc: s.respSvc, }, nil } diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go deleted file mode 100644 index 962f2bc26..000000000 --- a/pkg/services/util/response/client_stream.go +++ /dev/null @@ -1,48 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ClientMessageStreamer represents client-side message streamer -// that sets meta values to the response. -type ClientMessageStreamer struct { - srv *Service - - send util.RequestMessageWriter - - close util.ClientStreamCloser -} - -// Send calls send method of internal streamer. -func (s *ClientMessageStreamer) Send(ctx context.Context, req any) error { - if err := s.send(ctx, req); err != nil { - return fmt.Errorf("(%T) could not send the request: %w", s, err) - } - return nil -} - -// CloseAndRecv closes internal stream, receivers the response, -// sets meta values and returns the result. -func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.ResponseMessage, error) { - resp, err := s.close(ctx) - if err != nil { - return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) - } - - s.srv.SetMeta(resp) - - return resp, nil -} - -// CreateRequestStreamer wraps stream methods and returns ClientMessageStreamer instance. -func (s *Service) CreateRequestStreamer(sender util.RequestMessageWriter, closer util.ClientStreamCloser) *ClientMessageStreamer { - return &ClientMessageStreamer{ - srv: s, - send: sender, - close: closer, - } -} -- 2.45.2 From 71889234b772297fbc5b016d1d6ec384b6ae8d21 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 23 Jun 2023 11:25:08 +0300 Subject: [PATCH 146/233] [#449] tree: Allow reading requests signed by keys from allow list Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/tree/config.go | 21 +++++++++++++++++++++ cmd/frostfs-node/tree.go | 1 + pkg/services/tree/options.go | 12 ++++++++++++ pkg/services/tree/signature.go | 15 +++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/cmd/frostfs-node/config/tree/config.go b/cmd/frostfs-node/config/tree/config.go index f6087c53d..8a8919999 100644 --- a/cmd/frostfs-node/config/tree/config.go +++ b/cmd/frostfs-node/config/tree/config.go @@ -1,9 +1,11 @@ package treeconfig import ( + "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) const ( @@ -71,3 +73,22 @@ func (c TreeConfig) ReplicationWorkerCount() int { func (c TreeConfig) SyncInterval() time.Duration { return config.DurationSafe(c.cfg, "sync_interval") } + +// AuthorizedKeys parses and returns an array of "authorized_keys" config +// parameter from "tree" section. +// +// Returns an empty list if not set. +func (c TreeConfig) AuthorizedKeys() keys.PublicKeys { + authorizedKeysStr := config.StringSliceSafe(c.cfg, "authorized_keys") + authorizedKeys := make(keys.PublicKeys, 0, len(authorizedKeysStr)) + + for i := range authorizedKeysStr { + pub, err := keys.NewPublicKeyFromString(authorizedKeysStr[i]) + if err != nil { + panic(fmt.Errorf("could not parse Tree authorized key %s: %w", authorizedKeysStr[i], err)) + } + + authorizedKeys = append(authorizedKeys, pub) + } + return authorizedKeys +} diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index fffaa01d1..1f73b7956 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -56,6 +56,7 @@ func initTreeService(c *cfg) { tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService())) for _, srv := range c.cfgGRPC.servers { diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index bcaf21f92..a6e23c625 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,6 +33,7 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int + authorizedKeys [][]byte metrics MetricsRegister } @@ -124,3 +125,14 @@ func WithMetrics(v MetricsRegister) Option { c.metrics = v } } + +// WithAuthorizedKeys returns option to add list of public +// keys that have rights to use Tree service. +func WithAuthorizedKeys(keys keys.PublicKeys) Option { + return func(c *cfg) { + c.authorizedKeys = nil + for _, key := range keys { + c.authorizedKeys = append(c.authorizedKeys, key.Bytes()) + } + } +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 439912969..976fc8d07 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -51,6 +51,21 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op if err != nil { return err } + if op == acl.OpObjectGet { + // verify if the request for a client operation + // was signed by a key from authorized list. + // Operation must be one of READ. + sign := req.GetSignature() + if sign == nil { + return errors.New("missing signature") + } + var key = sign.GetKey() + for i := range s.authorizedKeys { + if bytes.Equal(s.authorizedKeys[i], key) { + return nil + } + } + } cnr, err := s.cnrSource.Get(cid) if err != nil { -- 2.45.2 From ab489265b3f27435a8a6e013fd2074319d9bf3e0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 21 Jun 2023 14:18:09 +0300 Subject: [PATCH 147/233] [#449] cli: Allow to exec `tree get-by-path` without bearer token Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/tree/get_by_path.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 6f6fc7ff9..69a3dc7ea 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -71,7 +71,9 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + } + if btok := common.ReadBearerToken(cmd, bearerFlagKey); btok != nil { + req.Body.BearerToken = btok.Marshal() } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) -- 2.45.2 From 72fedff7ad4d7c523bc6433025eda7e3f98ef093 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 16:33:05 +0300 Subject: [PATCH 148/233] [#426] cli: Add object nodes command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 358 ++++++++++++++++++++++++ cmd/frostfs-cli/modules/object/root.go | 4 +- 2 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-cli/modules/object/nodes.go diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go new file mode 100644 index 000000000..358a9d618 --- /dev/null +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -0,0 +1,358 @@ +package object + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + "strconv" + "sync" + "text/tabwriter" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +const ( + verifyPresenceAllFlag = "verify-presence-all" +) + +type objectNodesInfo struct { + containerID cid.ID + objectID oid.ID + relatedObjectIDs []oid.ID + isLock bool +} + +type boolError struct { + value bool + err error +} + +var objectNodesCmd = &cobra.Command{ + Use: "nodes", + Short: "List of nodes where the object is stored", + Long: `List of nodes where the object should be stored and where it is actually stored. + Lock objects must exist on all nodes of the container. + For complex objects, a node is considered to store an object if the node stores at least one part of the complex object. + By default, the actual storage of the object is checked only on the nodes that should store the object. To check all nodes, use the flag --verify-presence-all.`, + Run: objectNodes, +} + +func initObjectNodesCmd() { + commonflags.Init(objectNodesCmd) + + flags := objectNodesCmd.Flags() + + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.CIDFlag) + + flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) + + flags.Bool("verify-presence-all", false, "Verify the actual presence of the object on all netmap nodes") +} + +func objectNodes(cmd *cobra.Command, _ []string) { + var cnrID cid.ID + var objID oid.ID + readObjectAddress(cmd, &cnrID, &objID) + + pk := key.GetOrGenerate(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) + + objectInfo := getObjectInfo(cmd, cnrID, objID, cli, pk) + + placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) + + requiredPlacement := getRequiredPlacement(cmd, objectInfo, placementPolicy, netmap) + + actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objectInfo) + + printPlacement(cmd, netmap, requiredPlacement, actualPlacement) +} + +func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) *objectNodesInfo { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + prmHead.SetRawFlag(true) + + Prepare(cmd, &prmHead) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(cmd.Context(), prmHead) + if err == nil { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + isLock: res.Header().Type() == objectSDK.TypeLock, + } + } + + var errSplitInfo *objectSDK.SplitInfoError + + if !errors.As(err, &errSplitInfo) { + commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) + return nil + } + + splitInfo := errSplitInfo.SplitInfo() + + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } +} + +func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.Client) (placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) { + eg, egCtx := errgroup.WithContext(cmd.Context()) + eg.Go(func() (e error) { + placementPolicy, e = getPlacementPolicy(egCtx, cnrID, cli) + return + }) + eg.Go(func() (e error) { + netmap, e = getNetMap(egCtx, cli) + return + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", eg.Wait()) + return +} + +func getPlacementPolicy(ctx context.Context, cnrID cid.ID, cli *client.Client) (netmapSDK.PlacementPolicy, error) { + var prm internalclient.GetContainerPrm + prm.SetClient(cli) + prm.SetContainer(cnrID) + + res, err := internalclient.GetContainer(ctx, prm) + if err != nil { + return netmapSDK.PlacementPolicy{}, err + } + + return res.Container().PlacementPolicy(), nil +} + +func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, error) { + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + res, err := internalclient.NetMapSnapshot(ctx, prm) + if err != nil { + return nil, err + } + nm := res.NetMap() + return &nm, nil +} + +func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { + nodes := make(map[uint64]netmapSDK.NodeInfo) + placementBuilder := placement.NewNetworkMapBuilder(netmap) + placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) + for repIdx, rep := range placement { + numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) + var nodeIdx uint32 + for _, n := range rep { + if !objInfo.isLock && nodeIdx == numOfReplicas { //lock object should be on all container nodes + break + } + nodes[n.Hash()] = n + nodeIdx++ + } + } + + for _, relatedObjID := range objInfo.relatedObjectIDs { + placement, err = placementBuilder.BuildPlacement(objInfo.containerID, &relatedObjID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement for related object: %w", err) + for _, rep := range placement { + for _, n := range rep { + nodes[n.Hash()] = n + } + } + } + + return nodes +} + +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, + pk *ecdsa.PrivateKey, objInfo *objectNodesInfo) map[uint64]boolError { + result := make(map[uint64]boolError) + resultMtx := &sync.Mutex{} + + var candidates []netmapSDK.NodeInfo + checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag) + if checkAllNodes { + candidates = netmap.Nodes() + } else { + for _, n := range requiredPlacement { + candidates = append(candidates, n) + } + } + + var err error + eg, egCtx := errgroup.WithContext(cmd.Context()) + for _, cand := range candidates { + cand := cand + + eg.Go(func() error { + var cli *client.Client + cli, err = createClient(egCtx, cmd, cand, pk) + if err != nil { + resultMtx.Lock() + defer resultMtx.Unlock() + result[cand.Hash()] = boolError{err: err} + return nil + } + + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, objInfo.objectID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + + for _, rObjID := range objInfo.relatedObjectIDs { + rObjID := rObjID + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, rObjID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + } + return nil + }) + } + + egErr := eg.Wait() + if err != nil || egErr != nil { + if err == nil { + err = egErr + } + commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", err) + } + return result +} + +func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { + var cli *client.Client + var addresses []string + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + var lastErr error + for _, address := range addresses { + var networkAddr network.Address + lastErr = networkAddr.FromString(address) + if lastErr != nil { + continue + } + cli, lastErr = internalclient.GetSDKClient(ctx, cmd, pk, networkAddr) + if lastErr == nil { + break + } + } + if lastErr != nil { + return nil, lastErr + } + if cli == nil { + return nil, fmt.Errorf("failed to create client: no available endpoint") + } + return cli, nil +} + +func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + + Prepare(cmd, &prmHead) + prmHead.SetTTL(1) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(ctx, prmHead) + if err == nil && res != nil { + return true, nil + } + var notFound *apistatus.ObjectNotFound + var removed *apistatus.ObjectAlreadyRemoved + if errors.As(err, ¬Found) || errors.As(err, &removed) { + return false, nil + } + return false, err +} + +func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, actualPlacement map[uint64]boolError) { + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + defer func() { + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) + }() + fmt.Fprintln(w, "Netmap node\tShould contain object\tActually contains object\t") + for _, n := range netmap.Nodes() { + var address string + n.IterateNetworkEndpoints(func(s string) bool { + address = s + return s != "" + }) + _, required := requiredPlacement[n.Hash()] + actual, actualExists := actualPlacement[n.Hash()] + actualStr := "" + if actualExists { + if actual.err != nil { + actualStr = fmt.Sprintf("error: %v", actual.err) + } else { + actualStr = strconv.FormatBool(actual.value) + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t\n", address, strconv.FormatBool(required), actualStr) + } +} diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 23badc576..c46fc058a 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -30,7 +30,8 @@ func init() { objectHeadCmd, objectHashCmd, objectRangeCmd, - objectLockCmd} + objectLockCmd, + objectNodesCmd} Cmd.AddCommand(objectChildCommands...) @@ -47,4 +48,5 @@ func init() { initObjectHashCmd() initObjectRangeCmd() initCommandObjectLock() + initObjectNodesCmd() } -- 2.45.2 From dd3874eff1cf8a0111ce5567ee9f46a5829245d7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 14:23:33 +0300 Subject: [PATCH 149/233] [#447] pilorama: Add benchmark for create ops Signed-off-by: Evgenii Stratonikov --- .../pilorama/bench_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/local_object_storage/pilorama/bench_test.go diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go new file mode 100644 index 000000000..e729b9ea6 --- /dev/null +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -0,0 +1,55 @@ +package pilorama + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/stretchr/testify/require" +) + +func getTimestamp(reorder int, ts Timestamp) Timestamp { + base := ts / Timestamp(reorder) + rem := ts % Timestamp(reorder) + return base*Timestamp(reorder) + Timestamp(reorder) - rem +} + +func BenchmarkCreate(b *testing.B) { + // Use `os.TempDir` because we construct multiple times in the same test. + tmpDir, err := os.MkdirTemp(os.TempDir(), "*") + require.NoError(b, err) + + f := NewBoltForest( + WithPath(filepath.Join(tmpDir, "test.db")), + WithMaxBatchSize(runtime.GOMAXPROCS(0))) + require.NoError(b, f.Open(false)) + require.NoError(b, f.Init()) + b.Cleanup(func() { + require.NoError(b, f.Close()) + require.NoError(b, os.RemoveAll(tmpDir)) + }) + + cid := cidtest.ID() + treeID := "tree" + ctx := context.Background() + var index atomic.Int32 + index.Store(-1) + b.SetParallelism(2) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i := index.Add(1) + op := &Move{ + Meta: Meta{Time: getTimestamp(runtime.GOMAXPROCS(0)*2, Timestamp(i+1))}, + Child: Node(i + 1), + Parent: RootID, + } + if err := f.TreeApply(ctx, cid, treeID, op, true); err != nil { + b.FailNow() + } + } + }) +} -- 2.45.2 From b4ce0b0412fed1554528cf8758744c1204a2885d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 15:06:16 +0300 Subject: [PATCH 150/233] [#447] pilorama: Do not undo log for create ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ Create-8 36.48µ ± 11% 30.34µ ± 14% -16.84% (p=0.000 n=10) │ old │ new │ │ B/op │ B/op vs base │ Create-8 43.01Ki ± 4% 37.78Ki ± 5% -12.15% (p=0.000 n=10) │ old │ new │ │ allocs/op │ allocs/op vs base │ Create-8 166.0 ± 3% 146.0 ± 3% -12.05% (p=0.000 n=10) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 64 +++++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 5722c68aa..bccd640f1 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -1,6 +1,7 @@ package pilorama import ( + "encoding/binary" "sort" "sync" "time" @@ -49,10 +50,67 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) - b.operations = removeDuplicatesInPlace(b.operations) - var lm Move - return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + + // Our main use-case is addition of new items. In this case, + // we do not need to perform undo()/redo(), just do(). + // https://github.com/trvedata/move-op/blob/6c23447c12a7862ff31b7fc2205f6c90fbdb9dc0/proof/Move_Create.thy#L259 + // + // For this optimization to work we need to ensure three things: + // 1. The node itself is not yet in tree. + // 2. The node is not a parent. This case is not mentioned in the article, because + // they consider a "static order" (perform all CREATE operations before MOVE). + // We need this because if node _is_ a parent, we could violate (3) for some late operation. + // See TestForest_ApplySameOperation for details. + // 3. Parent of each operation is already in tree. + var parents map[uint64]struct{} + var cKey [17]byte + var slow bool + for i := range b.operations { + _, _, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], b.operations[i].Child)) + if inTree { + slow = true + break + } + + key := childrenKey(cKey[:], b.operations[i].Child, 0) + k, _ := bTree.Cursor().Seek(key) + if len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { + slow = true + break + } + + if b.operations[i].Parent == RootID { + continue + } else if parents == nil { + // Attaching key only to root is done frequently, + // no allocations are performed unless necessary. + parents = make(map[uint64]struct{}) + } else if _, ok := parents[b.operations[i].Parent]; ok { + continue + } + + p := b.operations[i].Parent + _, ts, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], p)) + if !inTree || b.operations[0].Time < ts { + slow = true + break + } + parents[b.operations[i].Parent] = struct{}{} + } + + if slow { + var lm Move + return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + } + + var key [17]byte + for i := range b.operations { + if err := b.forest.do(bLog, bTree, key[:], b.operations[i]); err != nil { + return err + } + } + return nil }) for i := range b.results { b.results[i] <- err -- 2.45.2 From 4bf345225c8dc98004cc79b6d9dd88008c936dd3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Jun 2023 13:39:18 +0300 Subject: [PATCH 151/233] [#447] pilorama: Use named constant for the key size Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 7 +++---- pkg/local_object_storage/pilorama/boltdb.go | 15 ++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index bccd640f1..c65488b74 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -64,7 +64,7 @@ func (b *batch) run() { // See TestForest_ApplySameOperation for details. // 3. Parent of each operation is already in tree. var parents map[uint64]struct{} - var cKey [17]byte + var cKey [maxKeySize]byte var slow bool for i := range b.operations { _, _, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], b.operations[i].Child)) @@ -75,7 +75,7 @@ func (b *batch) run() { key := childrenKey(cKey[:], b.operations[i].Child, 0) k, _ := bTree.Cursor().Seek(key) - if len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { + if len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { slow = true break } @@ -104,9 +104,8 @@ func (b *batch) run() { return b.forest.applyOperation(bLog, bTree, b.operations, &lm) } - var key [17]byte for i := range b.operations { - if err := b.forest.do(bLog, bTree, key[:], b.operations[i]); err != nil { + if err := b.forest.do(bLog, bTree, cKey[:], b.operations[i]); err != nil { return err } } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index a19561139..53a52433d 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -37,6 +37,11 @@ type boltForest struct { cfg } +const ( + childrenKeySize = 17 + maxKeySize = childrenKeySize +) + var ( dataBucket = []byte{0} logBucket = []byte{1} @@ -199,7 +204,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin if lm.Child == RootID { lm.Child = t.findSpareID(bTree) } - return t.do(bLog, bTree, make([]byte, 17), &lm) + return t.do(bLog, bTree, make([]byte, maxKeySize), &lm) })) } @@ -395,7 +400,7 @@ func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID stri } var lm []Move - var key [17]byte + var key [maxKeySize]byte fullID := bucketName(d.CID, treeID) err := t.db.Batch(func(tx *bbolt.Tx) error { @@ -612,7 +617,7 @@ func (t *boltForest) getTreeBuckets(tx *bbolt.Tx, treeRoot []byte) (*bbolt.Bucke // applyOperations applies log operations. Assumes lm are sorted by timestamp. func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*Move, lm *Move) error { var tmp Move - var cKey [17]byte + var cKey [maxKeySize]byte c := logBucket.Cursor() @@ -961,7 +966,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) c := b.Cursor() - for k, _ := c.Seek(key); len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { children = append(children, binary.LittleEndian.Uint64(k[9:])) } return nil @@ -1216,7 +1221,7 @@ func childrenKey(key []byte, child, parent Node) []byte { key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], parent) binary.LittleEndian.PutUint64(key[9:], child) - return key[:17] + return key[:childrenKeySize] } // 'i' + attribute name (string) + attribute value (string) + parent (id) + node (id) -> 0/1. -- 2.45.2 From 71a63b8e9c629fb7efb5343e108995296df97ed3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Jun 2023 10:07:38 +0300 Subject: [PATCH 152/233] [#439] node: Update SDK-Go version This fixed version: null header for complex object. Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | Bin 99899 -> 99899 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0fdcb4409..7a1c2f074 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 5da5a9a9ed7db66bf1919e0c23dc30541c154708..dfa8632aa1b0773da300effd7988e3d426c725c0 100644 GIT binary patch delta 179 zcmdnp!?wGJZG(lNi;=ORfr*8Ik#1t5MOs>FQnHz`k%dBrp;d}enQ^#5nNLb`VPr_8 zp?-itadJv|XlZt?Q(9SxwsTc>wu^ylS#frNRj6HNo|~SORitg?npAA2A5meEUg2F4;o+9%;Tjd{VeDt(3N%w;a*YsIv#!u~ JT_Hxk5&({qH$wmb -- 2.45.2 From cac4ed93d6a73e673864896453ab5197056e6458 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 22 Jun 2023 10:46:56 +0300 Subject: [PATCH 153/233] [#428] engine: Add low_mem config parameter Concurrent initialization in case of the metabase resync leads to high memory consumption and potential OOM. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 4 ++- cmd/frostfs-node/config/engine/config.go | 5 ++++ docs/storage-node-configuration.md | 1 + pkg/local_object_storage/engine/control.go | 29 +++++++++++++++++----- pkg/local_object_storage/engine/engine.go | 9 +++++++ pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +-- 7 files changed, 44 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2cb785d30..5d7adab29 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -98,6 +98,7 @@ type applicationConfiguration struct { errorThreshold uint32 shardPoolSize uint32 shards []shardCfg + lowMem bool } } @@ -200,6 +201,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) + a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } @@ -675,8 +677,8 @@ func (c *cfg) engineOpts() []engine.Option { opts = append(opts, engine.WithShardPoolSize(c.EngineCfg.shardPoolSize), engine.WithErrorThreshold(c.EngineCfg.errorThreshold), - engine.WithLogger(c.log), + engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), ) if c.metricsCollector != nil { diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index 36684f093..c944d1c58 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -83,3 +83,8 @@ func ShardPoolSize(c *config.Config) uint32 { func ShardErrorThreshold(c *config.Config) uint32 { return config.Uint32Safe(c.Sub(subsection), "shard_ro_error_threshold") } + +// EngineLowMemoryConsumption returns value of "lowmem" config parmeter from "storage" section. +func EngineLowMemoryConsumption(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "low_mem") +} diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 2c78cf6b1..4469b1e10 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -167,6 +167,7 @@ Local storage engine configuration. |----------------------------|-----------------------------------|---------------|------------------------------------------------------------------------------------------------------------------| | `shard_pool_size` | `int` | `20` | Pool size for shard workers. Limits the amount of concurrent `PUT` operations on each shard. | | `shard_ro_error_threshold` | `int` | `0` | Maximum amount of storage errors to encounter before shard automatically moves to `Degraded` or `ReadOnly` mode. | +| `low_mem` | `bool` | `false` | Reduce memory consumption by reducing performance. | | `shard` | [Shard config](#shard-subsection) | | Configuration for separate shards. | ## `shard` subsection diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 9ad4fcf9c..bd166b3ff 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) type shardInitError struct { @@ -74,23 +75,30 @@ func (e *StorageEngine) Init(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() - var wg sync.WaitGroup var errCh = make(chan shardInitError, len(e.shards)) + var eg errgroup.Group + if e.cfg.lowMem && e.anyShardRequiresRefill() { + eg.SetLimit(1) + } for id, sh := range e.shards { - wg.Add(1) - go func(id string, sh *shard.Shard) { - defer wg.Done() + id := id + sh := sh + eg.Go(func() error { if err := sh.Init(ctx); err != nil { errCh <- shardInitError{ err: err, id: id, } } - }(id, sh.Shard) + return nil + }) } - wg.Wait() + err := eg.Wait() close(errCh) + if err != nil { + return fmt.Errorf("failed to initialize shards: %w", err) + } for res := range errCh { if res.err != nil { @@ -125,6 +133,15 @@ func (e *StorageEngine) Init(ctx context.Context) error { return nil } +func (e *StorageEngine) anyShardRequiresRefill() bool { + for _, sh := range e.shards { + if sh.NeedRefillMetabase() { + return true + } + } + return false +} + var errClosed = errors.New("storage engine is closed") // Close releases all StorageEngine's components. Waits for all data-related operations to complete. diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 3061d6383..7bc84e6c9 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -210,6 +210,8 @@ type cfg struct { metrics MetricRegister shardPoolSize uint32 + + lowMem bool } func defaultCfg() *cfg { @@ -265,3 +267,10 @@ func WithErrorThreshold(sz uint32) Option { c.errorsThreshold = sz } } + +// WithLowMemoryConsumption returns an option to set the flag to reduce memory consumption by reducing performance. +func WithLowMemoryConsumption(lowMemCons bool) Option { + return func(c *cfg) { + c.lowMem = lowMemCons + } +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index bc514933b..db8248c02 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -98,7 +98,7 @@ func (s *Shard) Init(ctx context.Context) error { if !s.GetMode().NoMetabase() { var initMetabase initializer - if s.needRefillMetabase() { + if s.NeedRefillMetabase() { initMetabase = (*metabaseSynchronizer)(s) } else { initMetabase = s.metaBase diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 10c1acd40..05799d236 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -231,8 +231,8 @@ func (s *Shard) hasWriteCache() bool { return s.cfg.useWriteCache } -// needRefillMetabase returns true if metabase is needed to be refilled. -func (s *Shard) needRefillMetabase() bool { +// NeedRefillMetabase returns true if metabase is needed to be refilled. +func (s *Shard) NeedRefillMetabase() bool { return s.cfg.refillMetabase } -- 2.45.2 From 43d263c3d512914427fe6ac5fdcc845a61778ee1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 22 Jun 2023 10:50:06 +0300 Subject: [PATCH 154/233] [#428] linter: Fix unkeyed assignment Thanks to gopls. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index c2bae9772..4cd35a088 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -37,7 +37,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { b.Fatal(err) } _, err = te.ng.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, nil, - []pilorama.KeyValue{{pilorama.AttributeFilename, []byte(strconv.Itoa(i))}}) + []pilorama.KeyValue{{Key: pilorama.AttributeFilename, Value: []byte(strconv.Itoa(i))}}) if err != nil { b.Fatal(err) } -- 2.45.2 From 0c866f62c5821039fdfd3accbf8f5c8fa0443d42 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 27 Jun 2023 11:12:01 +0300 Subject: [PATCH 155/233] [#473] placement: Fix backwards compatibility for `copies_number` Signed-off-by: Evgenii Stratonikov --- pkg/services/object_manager/placement/traverser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 7b9b26a0b..47f79a627 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -86,7 +86,7 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } // backward compatibility for scalar `copies_number` - if len(cfg.copyNumbers) == 1 { + if len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] != 0 { cfg.flatSuccess = &cfg.copyNumbers[0] } -- 2.45.2 From 13b53258b46bf696f49715d6080bfb5b660f1284 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:51:01 +0300 Subject: [PATCH 156/233] [#1] go.mod: Remove retract version Signed-off-by: Evgenii Stratonikov --- go.mod | 5 ----- 1 file changed, 5 deletions(-) diff --git a/go.mod b/go.mod index 7a1c2f074..aca1d2d6f 100644 --- a/go.mod +++ b/go.mod @@ -119,8 +119,3 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -retract ( - v1.22.1 // Contains retraction only. - v1.22.0 // Published accidentally. -) -- 2.45.2 From 64e1383df604f635c77f7a646e4b198aaab7962c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:52:58 +0300 Subject: [PATCH 157/233] [#1] Makefile: Remove `image-storage-testnet` target Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.storage-testnet | 19 ------------------- Makefile | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 .docker/Dockerfile.storage-testnet diff --git a/.docker/Dockerfile.storage-testnet b/.docker/Dockerfile.storage-testnet deleted file mode 100644 index 28bbf7a1b..000000000 --- a/.docker/Dockerfile.storage-testnet +++ /dev/null @@ -1,19 +0,0 @@ -FROM golang:1.20 as builder -ARG BUILD=now -ARG VERSION=dev -ARG REPO=repository -WORKDIR /src -COPY . /src - -RUN make bin/frostfs-node - -# Executable image -FROM alpine AS frostfs-node -RUN apk add --no-cache bash - -WORKDIR / - -COPY --from=builder /src/bin/frostfs-node /bin/frostfs-node -COPY --from=builder /src/config/testnet/config.yml /config.yml - -CMD ["frostfs-node", "--config", "/config.yml"] diff --git a/Makefile b/Makefile index cf2095ec0..e1b719206 100755 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ image-%: -t $(HUB_IMAGE)-$*:$(HUB_TAG) . # Build all Docker images -images: image-storage image-ir image-cli image-adm image-storage-testnet +images: image-storage image-ir image-cli image-adm # Build dirty local Docker images dirty-images: image-dirty-storage image-dirty-ir image-dirty-cli image-dirty-adm -- 2.45.2 From f4c71cea659d6966f753ff9d0bc157012d616cb4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:58:22 +0300 Subject: [PATCH 158/233] [#1] *: Replace outdated FIXME/TODO links Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/list.go | 3 ++- cmd/frostfs-cli/modules/container/list_objects.go | 3 ++- pkg/core/object/fmt.go | 1 - pkg/innerring/processors/container/common.go | 3 --- pkg/local_object_storage/metabase/select.go | 1 - pkg/services/object/internal/client/client.go | 2 +- 6 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 1c9e4767f..189cc05c7 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -78,7 +78,8 @@ var listContainersCmd = &cobra.Command{ if flagVarListPrintAttr { cnr.IterateAttributes(func(key, val string) { if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", key, val) } }) diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index e417560e8..d00cc3576 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -71,7 +71,8 @@ var listContainerObjectsCmd = &cobra.Command{ for i := range attrs { attrKey := attrs[i].Key() if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) && !strings.HasPrefix(attrKey, v2object.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#226 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", attrKey, attrs[i].Value()) } } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 946cfc462..bfd22c9ae 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -133,7 +133,6 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr } func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { - // FIXME(@cthulhu-rider): temp solution, see neofs-sdk-go#233 sig := obj.Signature() if sig == nil { // TODO(@cthulhu-rider): #1387 use "const" error diff --git a/pkg/innerring/processors/container/common.go b/pkg/innerring/processors/container/common.go index 375e4c179..97eb6f559 100644 --- a/pkg/innerring/processors/container/common.go +++ b/pkg/innerring/processors/container/common.go @@ -63,7 +63,6 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { } if keyProvided { - // TODO(@cthulhu-rider): #1387 use another approach after neofs-sdk-go#233 var idFromKey user.ID user.IDFromKey(&idFromKey, (ecdsa.PublicKey)(key)) @@ -118,8 +117,6 @@ func (cp *Processor) verifyByTokenSession(v signatureVerificationData, key *fros return errors.New("invalid session token signature") } - // FIXME(@cthulhu-rider): #1387 check token is signed by container owner, see neofs-sdk-go#233 - if keyProvided && !tok.AssertAuthKey(key) { return errors.New("signed with a non-session key") } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 1f32d2cde..0f53a8f36 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -572,6 +572,5 @@ func blindlyProcess(fs object.SearchFilters) bool { // returns true if string key is a reserved system filter key. func isSystemKey(key string) bool { - // FIXME: #1147 version-dependent approach return strings.HasPrefix(key, v2object.ReservedFilterPrefix) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 8214c784d..17bf6a40f 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -81,7 +81,7 @@ type readPrmCommon struct { // // By default current epoch on the server will be used. func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { - // FIXME: (neofs-node#1194) not supported by client + // FIXME(@fyrchik): https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/465 } // GetObjectPrm groups parameters of GetObject operation. -- 2.45.2 From 73a71a71b06adb053933e7d7eab117ceb3406784 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 14:43:25 +0300 Subject: [PATCH 159/233] [#1] node: Use a proper validation of a substorage type Signed-off-by: Evgenii Stratonikov --- .../config/engine/shard/blobstor/config.go | 13 ++++--------- cmd/frostfs-node/validate.go | 6 +----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/config.go index a6e34e80f..f8b2e2e9b 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/config.go @@ -5,8 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/storage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" ) // Config is a wrapper over the config section @@ -25,14 +23,11 @@ func (x *Config) Storages() []*storage.Config { typ := config.String( (*config.Config)(x), strconv.Itoa(i)+".type") - switch typ { - case "": + if typ == "" { return ss - case fstree.Type, blobovniczatree.Type: - sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) - ss = append(ss, sub) - default: - panic("invalid type") } + + sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) + ss = append(ss, sub) } } diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index 3896fd6be..e07afb2ca 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -51,17 +51,13 @@ func validateConfig(c *config.Config) error { blobstor := sc.BlobStor().Storages() if len(blobstor) != 2 { - // TODO (@fyrcik): remove after #1522 return fmt.Errorf("blobstor section must have 2 components, got: %d", len(blobstor)) } for i := range blobstor { switch blobstor[i].Type() { case fstree.Type, blobovniczatree.Type: default: - // FIXME #1764 (@fyrchik): this line is currently unreachable, - // because we panic in `sc.BlobStor().Storages()`. - return fmt.Errorf("unexpected storage type: %s (shard %d)", - blobstor[i].Type(), shardNum) + return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } if blobstor[i].Perm()&0600 != 0600 { return fmt.Errorf("invalid permissions for blobstor component: %s, "+ -- 2.45.2 From cab51c8cbe959739b7808a6c797ff0f04b45ff56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 14:49:08 +0300 Subject: [PATCH 160/233] [#1] metabase: Rename blindlyProcess() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/select.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 0f53a8f36..998ce9b94 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -84,7 +84,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err return res, ErrDegradedMode } - if blindlyProcess(prm.filters) { + if checkNonEmpty(prm.filters) { success = true return res, nil } @@ -556,15 +556,12 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) { } } -// returns true if query leads to a deliberately empty result. -func blindlyProcess(fs object.SearchFilters) bool { +// Returns true if at least 1 object can satisfy fs. +func checkNonEmpty(fs object.SearchFilters) bool { for i := range fs { if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) { return true } - - // TODO: #1148 check other cases - // e.g. (a == b) && (a != b) } return false -- 2.45.2 From 4f413fe86e33836dc99b5d9629efaca27f924493 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 15:00:44 +0300 Subject: [PATCH 161/233] [#1] treesvc: Properly check for secure transport Signed-off-by: Evgenii Stratonikov --- pkg/network/address.go | 2 +- pkg/network/group.go | 2 +- pkg/network/tls.go | 4 ++-- pkg/network/tls_test.go | 2 +- pkg/services/tree/cache.go | 4 +--- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/network/address.go b/pkg/network/address.go index 020882980..8ad285725 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -47,7 +47,7 @@ func (a Address) URIAddr() string { panic(fmt.Errorf("could not get host addr: %w", err)) } - if !a.isTLSEnabled() { + if !a.IsTLSEnabled() { return host } diff --git a/pkg/network/group.go b/pkg/network/group.go index c18feac27..a6de0653e 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -57,7 +57,7 @@ func (x AddressGroup) Len() int { // Less returns true if i-th address in AddressGroup supports TLS // and j-th one doesn't. func (x AddressGroup) Less(i, j int) bool { - return x[i].isTLSEnabled() && !x[j].isTLSEnabled() + return x[i].IsTLSEnabled() && !x[j].IsTLSEnabled() } // Swap swaps i-th and j-th addresses in AddressGroup. diff --git a/pkg/network/tls.go b/pkg/network/tls.go index de2c93694..9aac89c47 100644 --- a/pkg/network/tls.go +++ b/pkg/network/tls.go @@ -11,8 +11,8 @@ const ( // tls var is used for (un)wrapping other multiaddrs around TLS multiaddr. var tls, _ = multiaddr.NewMultiaddr("/" + tlsProtocolName) -// isTLSEnabled searches for wrapped TLS protocol in multiaddr. -func (a Address) isTLSEnabled() bool { +// IsTLSEnabled searches for wrapped TLS protocol in multiaddr. +func (a Address) IsTLSEnabled() bool { for _, protoc := range a.ma.Protocols() { if protoc.Code == multiaddr.P_TLS { return true diff --git a/pkg/network/tls_test.go b/pkg/network/tls_test.go index 25775eaf1..d93ea6a12 100644 --- a/pkg/network/tls_test.go +++ b/pkg/network/tls_test.go @@ -24,6 +24,6 @@ func TestAddress_TLSEnabled(t *testing.T) { err := addr.FromString(test.input) require.NoError(t, err) - require.Equal(t, test.wantTLS, addr.isTLSEnabled(), test.input) + require.Equal(t, test.wantTLS, addr.IsTLSEnabled(), test.input) } } diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 97218da08..ef0c4b465 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" @@ -100,8 +99,7 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, ), } - // FIXME(@fyrchik): ugly hack #1322 - if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { + if netAddr.IsTLSEnabled() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } -- 2.45.2 From 8a4e250daefdd469ea0e59e33d587a0823b07dee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 16:18:39 +0300 Subject: [PATCH 162/233] [#468] *: replace outdated TODO crypto-related links Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/util.go | 2 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-node/object.go | 2 +- pkg/core/object/fmt.go | 2 +- pkg/morph/client/container/eacl.go | 2 +- pkg/morph/client/container/eacl_set.go | 2 +- pkg/morph/client/container/get.go | 2 +- pkg/morph/client/container/put.go | 2 +- pkg/services/container/morph/executor.go | 4 ++-- pkg/services/control/ir/server/sign.go | 8 ++++---- pkg/services/control/server/sign.go | 8 ++++---- pkg/services/object/acl/v2/util.go | 2 +- pkg/services/tree/signature.go | 2 +- 13 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index fdf17244b..5ad675c0e 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -40,7 +40,7 @@ func verifyResponse(cmd *cobra.Command, commonCmd.ExitOnErr(cmd, "", errors.New("missing response signature")) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetScheme(refs.ECDSA_SHA512) sigV2.SetKey(sigControl.GetKey()) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 6fce04490..04467744d 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -163,7 +163,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { if signature := obj.Signature(); signature != nil { cmd.Print("ID signature:\n") - // TODO(@carpawell): #1387 implement and use another approach to avoid conversion + // TODO(@carpawell): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature signature.WriteToV2(&sigV2) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 0476dbcac..afc4bcb41 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -435,7 +435,7 @@ func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { } if !eaclInfo.Signature.Verify(binTable) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("invalid signature of the eACL table") } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index bfd22c9ae..ba9868693 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -135,7 +135,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { sig := obj.Signature() if sig == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go index 56c36c179..54c83737d 100644 --- a/pkg/morph/client/container/eacl.go +++ b/pkg/morph/client/container/eacl.go @@ -86,7 +86,7 @@ func (c *Client) GetEACL(cnr cid.ID) (*container.EACL, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sig) diff --git a/pkg/morph/client/container/eacl_set.go b/pkg/morph/client/container/eacl_set.go index 86eae4c2b..2d2ffb456 100644 --- a/pkg/morph/client/container/eacl_set.go +++ b/pkg/morph/client/container/eacl_set.go @@ -31,7 +31,7 @@ func PutEACL(c *Client, eaclInfo containercore.EACL) error { prm.SetToken(eaclInfo.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature eaclInfo.Signature.WriteToV2(&sigV2) diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 009b22f3c..0513eea4b 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -105,7 +105,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sigBytes) diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 2c97446c6..5c23eb36d 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -28,7 +28,7 @@ func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { prm.SetToken(cnr.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature cnr.Signature.WriteToV2(&sigV2) diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index ae37da520..dec022219 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -52,7 +52,7 @@ func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *container.PutRequestBody) (*container.PutResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } @@ -206,7 +206,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) func (s *morphExecutor) SetExtendedACL(_ context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } diff --git a/pkg/services/control/ir/server/sign.go b/pkg/services/control/ir/server/sign.go index 4ada98468..f72d51f9e 100644 --- a/pkg/services/control/ir/server/sign.go +++ b/pkg/services/control/ir/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/control/server/sign.go b/pkg/services/control/server/sign.go index 726cdf341..acc405821 100644 --- a/pkg/services/control/server/sign.go +++ b/pkg/services/control/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index aa5d67584..cd45b63fc 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -118,7 +118,7 @@ func ownerFromToken(token *sessionSDK.Object) (*user.ID, *keys.PublicKey, error) } // 2. Then check if session token owner issued the session token - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var tokV2 sessionV2.Token token.WriteToV2(&tokV2) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 976fc8d07..08aa226fa 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -159,7 +159,7 @@ func verifyMessage(m message) error { sig := m.GetSignature() - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sig.GetKey()) sigV2.SetSign(sig.GetSign()) -- 2.45.2 From f91cfb36e6eb33bcdaa1f35dc5f77a58bbb98d9d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 11:09:19 +0300 Subject: [PATCH 163/233] [#478] .forgejo: Add build/test workflows Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/build.yml | 38 +++++++++++++++++++++++++ .forgejo/workflows/tests.yml | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 .forgejo/workflows/build.yml create mode 100644 .forgejo/workflows/tests.yml diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 000000000..5e4a97b93 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,38 @@ +name: Build + +on: [pull_request] + +jobs: + build: + name: Build Components + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + + steps: + - uses: actions/checkout@v3 + with: + # Allows to fetch all history for all branches and tags. + # Need this for proper versioning. + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + + - name: Build CLI + run: make bin/frostfs-cli + + - name: Build NODE + run: make bin/frostfs-node + + - name: Build IR + run: make bin/frostfs-ir + + - name: Build ADM + run: make bin/frostfs-adm + + - name: Build LENS + run: make bin/frostfs-lens diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml new file mode 100644 index 000000000..2d44d34f8 --- /dev/null +++ b/.forgejo/workflows/tests.yml @@ -0,0 +1,54 @@ +name: Tests and linters +on: [pull_request] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: golangci-lint + uses: https://github.com/golangci/golangci-lint-action@v3 + with: + version: latest + + tests: + name: Tests + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + cache: true + + - name: Run tests + run: make test + + tests-race: + name: Tests with -race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Run tests + run: go test ./... -count=1 -race -- 2.45.2 From 33d9ebbe7f70c639a7b45373262cda8b3dac00af Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 14:05:09 +0300 Subject: [PATCH 164/233] [#478] .golangci.yml: Increase timeout Make it enough for CI. Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index fae355a3d..4a3fec6e3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ # options for analysis running run: # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 10m + timeout: 20m # include test files or not, default is true tests: false -- 2.45.2 From d0ab552a90bc0359b159f558e7a7a36fbe0afdf7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 14:10:06 +0300 Subject: [PATCH 165/233] [#478] *: Fix funlen linter warnings Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 94 +++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 08aa226fa..63485a707 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -51,20 +51,10 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op if err != nil { return err } - if op == acl.OpObjectGet { - // verify if the request for a client operation - // was signed by a key from authorized list. - // Operation must be one of READ. - sign := req.GetSignature() - if sign == nil { - return errors.New("missing signature") - } - var key = sign.GetKey() - for i := range s.authorizedKeys { - if bytes.Equal(s.authorizedKeys[i], key) { - return nil - } - } + + isAuthorized, err := s.isAuthorized(req, op) + if isAuthorized || err != nil { + return err } cnr, err := s.cnrSource.Get(cid) @@ -74,18 +64,9 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op eaclOp := eACLOp(op) - var bt *bearer.Token - if len(rawBearer) > 0 { - bt = new(bearer.Token) - if err = bt.Unmarshal(rawBearer); err != nil { - return eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) - } - if !bt.AssertContainer(cid) { - return eACLErr(eaclOp, errBearerWrongContainer) - } - if !bt.VerifySignature() { - return eACLErr(eaclOp, errBearerSignature) - } + bt, err := parseBearer(rawBearer, cid, eaclOp) + if err != nil { + return err } role, err := roleFromReq(cnr, req, bt) @@ -117,32 +98,65 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op var tb eacl.Table signer := req.GetSignature().GetKey() - if tableFromBearer { - if bt.Impersonate() { - tbCore, err := s.eaclSource.GetEACL(cid) - if err != nil { - return handleGetEACLError(err) - } - tb = *tbCore.Value - signer = bt.SigningKeyBytes() - } else { - if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { - return eACLErr(eaclOp, errBearerWrongOwner) - } - tb = bt.EACLTable() + if tableFromBearer && !bt.Impersonate() { + if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { + return eACLErr(eaclOp, errBearerWrongOwner) } + tb = bt.EACLTable() } else { tbCore, err := s.eaclSource.GetEACL(cid) if err != nil { return handleGetEACLError(err) } - tb = *tbCore.Value + + if bt.Impersonate() { + signer = bt.SigningKeyBytes() + } } return checkEACL(tb, signer, eACLRole(role), eaclOp) } +// Returns true iff the operation is read-only and request was signed +// with one of the authorized keys. +func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { + if op != acl.OpObjectGet { + return false, nil + } + + sign := req.GetSignature() + if sign == nil { + return false, errors.New("missing signature") + } + + key := sign.GetKey() + for i := range s.authorizedKeys { + if bytes.Equal(s.authorizedKeys[i], key) { + return true, nil + } + } + return false, nil +} + +func parseBearer(rawBearer []byte, cid cidSDK.ID, eaclOp eacl.Operation) (*bearer.Token, error) { + if len(rawBearer) == 0 { + return nil, nil + } + + bt := new(bearer.Token) + if err := bt.Unmarshal(rawBearer); err != nil { + return nil, eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + } + if !bt.AssertContainer(cid) { + return nil, eACLErr(eaclOp, errBearerWrongContainer) + } + if !bt.VerifySignature() { + return nil, eACLErr(eaclOp, errBearerSignature) + } + return bt, nil +} + func handleGetEACLError(err error) error { if client.IsErrEACLNotFound(err) { return nil -- 2.45.2 From d01c064674758bb320c5970bb324e44c11bef011 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 27 Jun 2023 14:26:39 +0300 Subject: [PATCH 166/233] [#455] Fix non-informative message when parse node attributes Signed-off-by: Anton Nikiforov --- pkg/util/attributes/parser.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/util/attributes/parser.go b/pkg/util/attributes/parser.go index 8016c0160..f8cc97189 100644 --- a/pkg/util/attributes/parser.go +++ b/pkg/util/attributes/parser.go @@ -1,7 +1,6 @@ package attributes import ( - "errors" "fmt" "strings" @@ -21,7 +20,7 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { k, v, found := strings.Cut(line, keyValueSeparator) if !found { - return errors.New("missing attribute key and/or value") + return fmt.Errorf("wrong format for node attribute: '%s'", attrs[i]) } _, ok := cache[k] @@ -36,9 +35,9 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { v = replaceEscaping(v, true) if k == "" { - return errors.New("empty key") + return fmt.Errorf("empty key in node attribute: '%s'", attrs[i]) } else if v == "" { - return errors.New("empty value") + return fmt.Errorf("empty value in node attribute: '%s'", attrs[i]) } dst.SetAttribute(k, v) -- 2.45.2 From f437ab8f15800b2b1baaca87afd3d0aa29e49a39 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 7 Jun 2023 13:04:55 +0300 Subject: [PATCH 167/233] [#197] object: Make Delete method return correct status Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/object/delete/exec.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 91bc6b3d7..343efbc04 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -75,8 +76,9 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { } func (exec *execCtx) formSplitInfo(ctx context.Context) bool { - var err error + success := false + var err error exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) switch { @@ -87,12 +89,17 @@ func (exec *execCtx) formSplitInfo(ctx context.Context) bool { exec.log.Debug(logs.DeleteCouldNotComposeSplitInfo, zap.String("error", err.Error()), ) - case err == nil: + case err == nil, apiclient.IsErrObjectAlreadyRemoved(err): + // IsErrObjectAlreadyRemoved check is required because splitInfo + // implicitly performs Head request that may return ObjectAlreadyRemoved + // status that is not specified for Delete + exec.status = statusOK exec.err = nil + success = true } - return err == nil + return success } func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { -- 2.45.2 From 0c5b0257889bf13743fe30f615f9707676ad0bcd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Jun 2023 18:27:42 +0300 Subject: [PATCH 168/233] [#470] grpc: Increase message limits For send message limit set to 2GiB, but there are custom GET/GET RANGE limiters. For receive message limit set to 256 MiB, but actual chunk size will be managed by client. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/grpc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index b62ae9c45..1dd0f0729 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -17,11 +17,13 @@ import ( "google.golang.org/grpc/credentials" ) +const maxRecvMsgSize = 256 << 20 + func initGRPC(c *cfg) { var successCount int grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { serverOpts := []grpc.ServerOption{ - grpc.MaxSendMsgSize(maxMsgSize), + grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.ChainUnaryInterceptor( metrics.NewUnaryServerInterceptor(), tracing.NewUnaryServerInterceptor(), -- 2.45.2 From f9730f090d1e9066d423f4a7b291bc9ffbb17a6b Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 29 Jun 2023 12:13:01 +0300 Subject: [PATCH 169/233] [#92] Refactor policer and add some unit tests Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/keyspaceiterator.go | 28 +++ cmd/frostfs-node/object.go | 22 +- pkg/services/control/server/evacuate.go | 11 +- pkg/services/policer/check.go | 88 ++----- pkg/services/policer/check_test.go | 8 +- pkg/services/policer/nodecache.go | 57 +++++ pkg/services/policer/option.go | 185 +++++++++++++++ pkg/services/policer/policer.go | 149 ------------ pkg/services/policer/policer_test.go | 299 ++++++++++++++++++++++++ pkg/services/policer/process.go | 18 +- pkg/services/policer/queue.go | 26 --- pkg/services/replicator/process.go | 24 +- pkg/services/replicator/task.go | 35 +-- 13 files changed, 640 insertions(+), 310 deletions(-) create mode 100644 cmd/frostfs-node/keyspaceiterator.go create mode 100644 pkg/services/policer/nodecache.go create mode 100644 pkg/services/policer/option.go create mode 100644 pkg/services/policer/policer_test.go delete mode 100644 pkg/services/policer/queue.go diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go new file mode 100644 index 000000000..8991964a0 --- /dev/null +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "fmt" + + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" +) + +type keySpaceIterator struct { + ng *engine.StorageEngine + cur *engine.Cursor +} + +func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.AddressWithType, error) { + var prm engine.ListWithCursorPrm + prm.WithCursor(it.cur) + prm.WithCount(batchSize) + + res, err := it.ng.ListWithCursor(ctx, prm) + if err != nil { + return nil, fmt.Errorf("cannot list objects in engine: %w", err) + } + + it.cur = res.Cursor() + return res.AddressList(), nil +} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index afc4bcb41..4106f5dc1 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -38,6 +38,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -211,15 +212,30 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl ls := c.cfgObject.cfgLocalStorage.localStorage + buryFn := func(ctx context.Context, addr oid.Address) error { + var prm engine.InhumePrm + prm.MarkAsGarbage(addr) + prm.WithForceRemoval() + + _, err := ls.Inhume(ctx, prm) + return err + } + + remoteHeader := headsvc.NewRemoteHeader(keyStorage, clientConstructor) + pol := policer.New( policer.WithLogger(c.log), - policer.WithLocalStorage(ls), + policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}), + policer.WithBuryFunc(buryFn), policer.WithContainerSource(c.cfgObject.cnrSource), policer.WithPlacementBuilder( placement.NewNetworkMapSourceBuilder(c.netMapSource), ), - policer.WithRemoteHeader( - headsvc.NewRemoteHeader(keyStorage, clientConstructor), + policer.WithRemoteObjectHeaderFunc( + func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a) + return remoteHeader.Head(ctx, prm) + }, ), policer.WithNetmapKeys(c), policer.WithHeadTimeout( diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index fc6dd3f60..8f62c3489 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -84,11 +84,12 @@ func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK } var res replicatorResult - var task replicator.Task - task.SetObject(obj) - task.SetObjectAddress(addr) - task.SetCopiesNumber(1) - task.SetNodes(nodes) + task := replicator.Task{ + NumCopies: 1, + Addr: addr, + Obj: obj, + Nodes: nodes, + } s.replicator.HandleTask(ctx, task, &res) if res.count == 0 { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index e91b8871b..db297b68a 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -7,8 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -18,55 +16,6 @@ import ( "go.uber.org/zap" ) -// tracks Policer's check progress. -type nodeCache map[uint64]bool - -func newNodeCache() *nodeCache { - m := make(map[uint64]bool) - return (*nodeCache)(&m) -} - -func (n *nodeCache) set(node netmap.NodeInfo, val bool) { - (*n)[node.Hash()] = val -} - -// submits storage node as a candidate to store the object replica in case of -// shortage. -func (n *nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { - n.set(node, false) -} - -// submits storage node as a current object replica holder. -func (n *nodeCache) submitReplicaHolder(node netmap.NodeInfo) { - n.set(node, true) -} - -// processStatus returns current processing status of the storage node -// -// >0 if node does not currently hold the object -// 0 if node already holds the object -// <0 if node has not been processed yet -func (n *nodeCache) processStatus(node netmap.NodeInfo) int8 { - val, ok := (*n)[node.Hash()] - if !ok { - return -1 - } - - if val { - return 0 - } - - return 1 -} - -// SubmitSuccessfulReplication marks given storage node as a current object -// replica holder. -// -// SubmitSuccessfulReplication implements replicator.TaskResult. -func (n *nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { - n.submitReplicaHolder(node) -} - func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) { addr := addrWithType.Address idCnr := addr.Container() @@ -79,11 +28,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.String("error", err.Error()), ) if container.IsErrNotFound(err) { - var prm engine.InhumePrm - prm.MarkAsGarbage(addrWithType.Address) - prm.WithForceRemoval() - - _, err := p.jobQueue.localStorage.Inhume(ctx, prm) + err := p.buryFn(ctx, addrWithType.Address) if err != nil { p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, zap.Stringer("cid", idCnr), @@ -145,10 +90,9 @@ type placementRequirements struct { } func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, - nodes []netmap.NodeInfo, shortage uint32, checkedNodes *nodeCache) { + nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache) { addr := addrWithType.Address typ := addrWithType.Type - prm := new(headsvc.RemoteHeadPrm).WithObjectAddress(addr) // Number of copies that are stored on maintenance nodes. var uncheckedCopies int @@ -175,8 +119,8 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi } else if nodes[i].IsMaintenance() { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { - if status := checkedNodes.processStatus(nodes[i]); status >= 0 { - if status == 0 { + if status := checkedNodes.processStatus(nodes[i]); status.Processed() { + if status == nodeHoldsObject { // node already contains replica, no need to replicate nodes = append(nodes[:i], nodes[i+1:]...) i-- @@ -188,7 +132,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - _, err := p.remoteHeader.Head(callCtx, prm.WithNodeInfo(nodes[i])) + _, err := p.remoteHeader(callCtx, nodes[i], addr) cancel() @@ -224,7 +168,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // prevent spam with new replicas. // However, additional copies should not be removed in this case, // because we can remove the only copy this way. -func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { +func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { checkedNodes.submitReplicaHolder(node) shortage-- uncheckedCopies++ @@ -236,25 +180,29 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCach } func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, - nodes []netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) { - if shortage > 0 { + nodes []netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) { + switch { + case shortage > 0: p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", addr), zap.Uint32("shortage", shortage), ) - var task replicator.Task - task.SetObjectAddress(addr) - task.SetNodes(nodes) - task.SetCopiesNumber(shortage) + task := replicator.Task{ + NumCopies: shortage, + Addr: addr, + Nodes: nodes, + } p.replicator.HandleTask(ctx, task, checkedNodes) - } else if uncheckedCopies > 0 { + + case uncheckedCopies > 0: // If we have more copies than needed, but some of them are from the maintenance nodes, // save the local copy. p.log.Debug(logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, zap.Int("count", uncheckedCopies)) - } else if uncheckedCopies == 0 { + + case uncheckedCopies == 0: // Safe to remove: checked all copies, shortage == 0. requirements.removeLocalCopy = true } diff --git a/pkg/services/policer/check_test.go b/pkg/services/policer/check_test.go index b40ee90d2..d4c7ccbf9 100644 --- a/pkg/services/policer/check_test.go +++ b/pkg/services/policer/check_test.go @@ -11,14 +11,14 @@ func TestNodeCache(t *testing.T) { cache := newNodeCache() node := netmaptest.NodeInfo() - require.Negative(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeNotProcessed) cache.SubmitSuccessfulReplication(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) cache.submitReplicaCandidate(node) - require.Positive(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeDoesNotHoldObject) cache.submitReplicaHolder(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) } diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go new file mode 100644 index 000000000..cd47cb0fc --- /dev/null +++ b/pkg/services/policer/nodecache.go @@ -0,0 +1,57 @@ +package policer + +import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + +type nodeProcessStatus int8 + +const ( + nodeNotProcessed nodeProcessStatus = iota + nodeDoesNotHoldObject + nodeHoldsObject +) + +func (st nodeProcessStatus) Processed() bool { + return st != nodeNotProcessed +} + +// nodeCache tracks Policer's check progress. +type nodeCache map[uint64]bool + +func newNodeCache() nodeCache { + return make(map[uint64]bool) +} + +func (n nodeCache) set(node netmap.NodeInfo, val bool) { + n[node.Hash()] = val +} + +// submits storage node as a candidate to store the object replica in case of +// shortage. +func (n nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { + n.set(node, false) +} + +// submits storage node as a current object replica holder. +func (n nodeCache) submitReplicaHolder(node netmap.NodeInfo) { + n.set(node, true) +} + +// processStatus returns current processing status of the storage node. +func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { + switch val, ok := n[node.Hash()]; { + case !ok: + return nodeNotProcessed + case val: + return nodeHoldsObject + default: + return nodeDoesNotHoldObject + } +} + +// SubmitSuccessfulReplication marks given storage node as a current object +// replica holder. +// +// SubmitSuccessfulReplication implements replicator.TaskResult. +func (n nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { + n.submitReplicaHolder(node) +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go new file mode 100644 index 000000000..6f17b2947 --- /dev/null +++ b/pkg/services/policer/option.go @@ -0,0 +1,185 @@ +package policer + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "go.uber.org/zap" +) + +// KeySpaceIterator is the interface that allows iterating over the key space +// of local storage. +// Note that the underlying implementation might be circular: i.e. it can restart +// when the end of the key space is reached. +type KeySpaceIterator interface { + Next(context.Context, uint32) ([]objectcore.AddressWithType, error) +} + +// RedundantCopyCallback is a callback to pass +// the redundant local copy of the object. +type RedundantCopyCallback func(context.Context, oid.Address) + +// BuryFunc is the function to bury (i.e. inhume) an object. +type BuryFunc func(context.Context, oid.Address) error + +// Replicator is the interface to a consumer of replication tasks. +type Replicator interface { + HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) +} + +// RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. +type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) + +// NodeLoader provides application load statistics. +type nodeLoader interface { + // ObjectServiceLoad returns object service load value in [0:1] range. + ObjectServiceLoad() float64 +} + +type cfg struct { + headTimeout time.Duration + + log *logger.Logger + + keySpaceIterator KeySpaceIterator + + buryFn BuryFunc + + cnrSrc container.Source + + placementBuilder placement.Builder + + remoteHeader RemoteObjectHeaderFunc + + netmapKeys netmap.AnnouncedKeys + + replicator Replicator + + cbRedundantCopy RedundantCopyCallback + + taskPool *ants.Pool + + loader nodeLoader + + maxCapacity int + + batchSize, cacheSize uint32 + + rebalanceFreq, evictDuration time.Duration +} + +func defaultCfg() *cfg { + return &cfg{ + log: &logger.Logger{Logger: zap.L()}, + batchSize: 10, + cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB + rebalanceFreq: 1 * time.Second, + evictDuration: 30 * time.Second, + } +} + +// Option is an option for Policer constructor. +type Option func(*cfg) + +// WithHeadTimeout returns option to set Head timeout of Policer. +func WithHeadTimeout(v time.Duration) Option { + return func(c *cfg) { + c.headTimeout = v + } +} + +// WithLogger returns option to set Logger of Policer. +func WithLogger(v *logger.Logger) Option { + return func(c *cfg) { + c.log = v + } +} + +func WithKeySpaceIterator(it KeySpaceIterator) Option { + return func(c *cfg) { + c.keySpaceIterator = it + } +} + +func WithBuryFunc(f BuryFunc) Option { + return func(c *cfg) { + c.buryFn = f + } +} + +// WithContainerSource returns option to set container source of Policer. +func WithContainerSource(v container.Source) Option { + return func(c *cfg) { + c.cnrSrc = v + } +} + +// WithPlacementBuilder returns option to set object placement builder of Policer. +func WithPlacementBuilder(v placement.Builder) Option { + return func(c *cfg) { + c.placementBuilder = v + } +} + +// WithRemoteObjectHeader returns option to set object header receiver of Policer. +func WithRemoteObjectHeaderFunc(v RemoteObjectHeaderFunc) Option { + return func(c *cfg) { + c.remoteHeader = v + } +} + +// WithNetmapKeys returns option to set tool to work with announced public keys. +func WithNetmapKeys(v netmap.AnnouncedKeys) Option { + return func(c *cfg) { + c.netmapKeys = v + } +} + +// WithReplicator returns option to set object replicator of Policer. +func WithReplicator(v Replicator) Option { + return func(c *cfg) { + c.replicator = v + } +} + +// WithRedundantCopyCallback returns option to set +// callback to pass redundant local object copies +// detected by Policer. +func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { + return func(c *cfg) { + c.cbRedundantCopy = cb + } +} + +// WithMaxCapacity returns option to set max capacity +// that can be set to the pool. +func WithMaxCapacity(capacity int) Option { + return func(c *cfg) { + c.maxCapacity = capacity + } +} + +// WithPool returns option to set pool for +// policy and replication operations. +func WithPool(p *ants.Pool) Option { + return func(c *cfg) { + c.taskPool = p + } +} + +// WithNodeLoader returns option to set FrostFS node load source. +func WithNodeLoader(l nodeLoader) Option { + return func(c *cfg) { + c.loader = l + } +} diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 541ab599c..4be13e0d8 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,29 +1,15 @@ package policer import ( - "context" "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" - "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) -// NodeLoader provides application load statistics. -type nodeLoader interface { - // ObjectServiceLoad returns object service load value in [0:1] range. - ObjectServiceLoad() float64 -} - type objectsInWork struct { m sync.RWMutex objs map[oid.Address]struct{} @@ -59,53 +45,6 @@ type Policer struct { objsInWork *objectsInWork } -// Option is an option for Policer constructor. -type Option func(*cfg) - -// RedundantCopyCallback is a callback to pass -// the redundant local copy of the object. -type RedundantCopyCallback func(context.Context, oid.Address) - -type cfg struct { - headTimeout time.Duration - - log *logger.Logger - - jobQueue jobQueue - - cnrSrc container.Source - - placementBuilder placement.Builder - - remoteHeader *headsvc.RemoteHeader - - netmapKeys netmap.AnnouncedKeys - - replicator *replicator.Replicator - - cbRedundantCopy RedundantCopyCallback - - taskPool *ants.Pool - - loader nodeLoader - - maxCapacity int - - batchSize, cacheSize uint32 - - rebalanceFreq, evictDuration time.Duration -} - -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - batchSize: 10, - cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB - rebalanceFreq: 1 * time.Second, - evictDuration: 30 * time.Second, - } -} - // New creates, initializes and returns Policer instance. func New(opts ...Option) *Policer { c := defaultCfg() @@ -129,91 +68,3 @@ func New(opts ...Option) *Policer { }, } } - -// WithHeadTimeout returns option to set Head timeout of Policer. -func WithHeadTimeout(v time.Duration) Option { - return func(c *cfg) { - c.headTimeout = v - } -} - -// WithLogger returns option to set Logger of Policer. -func WithLogger(v *logger.Logger) Option { - return func(c *cfg) { - c.log = v - } -} - -// WithLocalStorage returns option to set local object storage of Policer. -func WithLocalStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.jobQueue.localStorage = v - } -} - -// WithContainerSource returns option to set container source of Policer. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -// WithPlacementBuilder returns option to set object placement builder of Policer. -func WithPlacementBuilder(v placement.Builder) Option { - return func(c *cfg) { - c.placementBuilder = v - } -} - -// WithRemoteHeader returns option to set object header receiver of Policer. -func WithRemoteHeader(v *headsvc.RemoteHeader) Option { - return func(c *cfg) { - c.remoteHeader = v - } -} - -// WithNetmapKeys returns option to set tool to work with announced public keys. -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -// WithReplicator returns option to set object replicator of Policer. -func WithReplicator(v *replicator.Replicator) Option { - return func(c *cfg) { - c.replicator = v - } -} - -// WithRedundantCopyCallback returns option to set -// callback to pass redundant local object copies -// detected by Policer. -func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { - return func(c *cfg) { - c.cbRedundantCopy = cb - } -} - -// WithMaxCapacity returns option to set max capacity -// that can be set to the pool. -func WithMaxCapacity(capacity int) Option { - return func(c *cfg) { - c.maxCapacity = capacity - } -} - -// WithPool returns option to set pool for -// policy and replication operations. -func WithPool(p *ants.Pool) Option { - return func(c *cfg) { - c.taskPool = p - } -} - -// WithNodeLoader returns option to set FrostFS node load source. -func WithNodeLoader(l nodeLoader) Option { - return func(c *cfg) { - c.loader = l - } -} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go new file mode 100644 index 000000000..a09957895 --- /dev/null +++ b/pkg/services/policer/policer_test.go @@ -0,0 +1,299 @@ +package policer + +import ( + "bytes" + "context" + "errors" + "sort" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" +) + +func TestBuryObjectWithoutContainer(t *testing.T) { + // Key space + addr := oidtest.Address() + objs := []objectcore.AddressWithType{ + { + Address: addr, + Type: object.TypeRegular, + }, + } + + // Container source and bury function + buryCh := make(chan oid.Address) + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + buryCh <- a + return nil + } + + // Task pool + pool, err := ants.NewPool(4) + require.NoError(t, err) + + // Policer instance + p := New( + WithKeySpaceIterator(&sliceKeySpaceIterator{objs: objs}), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + WithNodeLoader(constNodeLoader(0)), + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Run(ctx) + + require.Equal(t, addr, <-buryCh) +} + +func TestProcessObject(t *testing.T) { + // Notes: + // - nodes are referred to by their index throughout, which is embedded in the public key + // - node with index 0 always refers to the local node, so there's no need to add it to objHolders + // - policy is used only to match the number of replicas for each index in the placement + tests := []struct { + desc string + objType object.Type + nodeCount int + policy string + placement [][]int + objHolders []int + maintenanceNodes []int + wantRemoveRedundant bool + wantReplicateTo []int + }{ + { + desc: "1 copy already held by local node", + nodeCount: 1, + policy: `REP 1`, + placement: [][]int{{0}}, + }, + { + desc: "1 copy already held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + objHolders: []int{1}, + wantRemoveRedundant: true, + }, + { + desc: "1 copy not yet held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "2 copies already held by local and remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + objHolders: []int{1}, + }, + { + desc: "2 copies but not held by remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "multiple vectors already held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + objHolders: []int{1}, + }, + { + desc: "multiple vectors not yet held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + wantReplicateTo: []int{1, 1}, // is this actually good? + }, + { + desc: "lock object must be replicated to all nodes", + objType: object.TypeLock, + nodeCount: 3, + policy: `REP 1`, + placement: [][]int{{0, 1, 2}}, + wantReplicateTo: []int{1, 2}, + }, + { + desc: "preserve local copy when maintenance nodes exist", + nodeCount: 3, + policy: `REP 2`, + placement: [][]int{{1, 2}}, + objHolders: []int{1}, + maintenanceNodes: []int{2}, + }, + } + + for i := range tests { + ti := tests[i] + t.Run(ti.desc, func(t *testing.T) { + addr := oidtest.Address() + + // Netmap, placement policy and placement builder + nodes := make([]netmap.NodeInfo, ti.nodeCount) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + for _, i := range ti.maintenanceNodes { + nodes[i].SetMaintenance() + } + + var policy netmap.PlacementPolicy + require.NoError(t, policy.DecodeString(ti.policy)) + + placementVectors := make([][]netmap.NodeInfo, len(ti.placement)) + for i, pv := range ti.placement { + for _, nj := range pv { + placementVectors[i] = append(placementVectors[i], nodes[nj]) + } + } + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + if cnr.Equals(addr.Container()) && obj != nil && obj.Equals(addr.Object()) { + return placementVectors, nil + } + t.Errorf("unexpected placement build: cid=%v oid=%v", cnr, obj) + return nil, errors.New("unexpected placement build") + } + + // Object remote header + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*object.Object, error) { + index := int(ni.PublicKey()[0]) + if a != addr || index < 1 || index >= ti.nodeCount { + t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) + return nil, errors.New("unexpected object head") + } + for _, i := range ti.objHolders { + if index == i { + return nil, nil + } + } + return nil, apistatus.ObjectNotFound{} + } + + // Container source + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := func(id cid.ID) (*container.Container, error) { + if id.Equals(addr.Container()) { + return cnr, nil + } + t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + t.Errorf("unexpected object buried: %v", a) + return nil + } + + // Policer instance + var gotRemoveRedundant bool + var gotReplicateTo []int + + p := New( + WithContainerSource(containerSrcFunc(containerSrc)), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithBuryFunc(buryFn), + WithRedundantCopyCallback(func(_ context.Context, a oid.Address) { + require.True(t, eqAddr(a, addr), "unexpected redundant copy callback: a=%v", a) + gotRemoveRedundant = true + }), + WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { + require.True(t, eqAddr(task.Addr, addr), "unexpected replicator task: %+v", task) + for _, node := range task.Nodes { + gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) + } + })), + ) + + addrWithType := objectcore.AddressWithType{ + Address: addr, + Type: ti.objType, + } + + p.processObject(context.Background(), addrWithType) + sort.Ints(gotReplicateTo) + + require.Equal(t, ti.wantRemoveRedundant, gotRemoveRedundant) + require.Equal(t, ti.wantReplicateTo, gotReplicateTo) + }) + } +} + +// TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) +func eqAddr(a, b oid.Address) bool { + return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) +} + +// sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. +type sliceKeySpaceIterator struct { + objs []objectcore.AddressWithType + cur int +} + +func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.cur >= len(it.objs) { + it.cur = 0 + return nil, engine.ErrEndOfListing + } + end := it.cur + int(size) + if end > len(it.objs) { + end = len(it.objs) + } + ret := it.objs[it.cur:end] + it.cur = end + return ret, nil +} + +// containerSrcFunc is a container.Source backed by a function. +type containerSrcFunc func(cid.ID) (*container.Container, error) + +func (f containerSrcFunc) Get(id cid.ID) (*container.Container, error) { return f(id) } + +// placementBuilderFunc is a placement.Builder backed by a function +type placementBuilderFunc func(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) + +func (f placementBuilderFunc) BuildPlacement(c cid.ID, o *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + return f(c, o, p) +} + +// announcedKeysFunc is a netmap.AnnouncedKeys backed by a function. +type announcedKeysFunc func([]byte) bool + +func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } + +// constNodeLoader is a nodeLoader that always returns a fixed value. +type constNodeLoader float64 + +func (f constNodeLoader) ObjectServiceLoad() float64 { return float64(f) } + +// replicatorFunc is a Replicator backed by a function. +type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) + +func (f replicatorFunc) HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { + f(ctx, task, res) +} diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 60e924755..6bd38f618 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -6,27 +6,17 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "go.uber.org/zap" ) func (p *Policer) Run(ctx context.Context) { - defer func() { - p.log.Info(logs.PolicerRoutineStopped) - }() - go p.poolCapacityWorker(ctx) p.shardPolicyWorker(ctx) + p.log.Info(logs.PolicerRoutineStopped) } func (p *Policer) shardPolicyWorker(ctx context.Context) { - var ( - addrs []objectcore.AddressWithType - cursor *engine.Cursor - err error - ) - for { select { case <-ctx.Done(): @@ -34,7 +24,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { default: } - addrs, cursor, err = p.jobQueue.Select(ctx, cursor, p.batchSize) + addrs, err := p.keySpaceIterator.Next(ctx, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { time.Sleep(time.Second) // finished whole cycle, sleep a bit @@ -55,7 +45,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { continue } - err = p.taskPool.Submit(func() { + err := p.taskPool.Submit(func() { v, ok := p.cache.Get(addr.Address) if ok && time.Since(v) < p.evictDuration { return @@ -78,10 +68,10 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { func (p *Policer) poolCapacityWorker(ctx context.Context) { ticker := time.NewTicker(p.rebalanceFreq) + defer ticker.Stop() for { select { case <-ctx.Done(): - ticker.Stop() return case <-ticker.C: frostfsSysLoad := p.loader.ObjectServiceLoad() diff --git a/pkg/services/policer/queue.go b/pkg/services/policer/queue.go deleted file mode 100644 index 22012c835..000000000 --- a/pkg/services/policer/queue.go +++ /dev/null @@ -1,26 +0,0 @@ -package policer - -import ( - "context" - "fmt" - - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" -) - -type jobQueue struct { - localStorage *engine.StorageEngine -} - -func (q *jobQueue) Select(ctx context.Context, cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { - var prm engine.ListWithCursorPrm - prm.WithCursor(cursor) - prm.WithCount(count) - - res, err := q.localStorage.ListWithCursor(ctx, prm) - if err != nil { - return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err) - } - - return res.AddressList(), res.Cursor(), nil -} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 0f82ff232..a54668e12 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -24,16 +24,16 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) defer p.metrics.DecInFlightRequest() defer func() { p.log.Debug(logs.ReplicatorFinishWork, - zap.Uint32("amount of unfinished replicas", task.quantity), + zap.Uint32("amount of unfinished replicas", task.NumCopies), ) }() - if task.obj == nil { + if task.Obj == nil { var err error - task.obj, err = engine.Get(ctx, p.localStorage, task.addr) + task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) if err != nil { p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, - zap.Stringer("object", task.addr), + zap.Stringer("object", task.Addr), zap.Error(err)) return @@ -41,9 +41,9 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } prm := new(putsvc.RemotePutPrm). - WithObject(task.obj) + WithObject(task.Obj) - for i := 0; task.quantity > 0 && i < len(task.nodes); i++ { + for i := 0; task.NumCopies > 0 && i < len(task.Nodes); i++ { select { case <-ctx.Done(): return @@ -51,13 +51,13 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } log := p.log.With( - zap.String("node", netmap.StringifyPublicKey(task.nodes[i])), - zap.Stringer("object", task.addr), + zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), + zap.Stringer("object", task.Addr), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) - err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.nodes[i])) + err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.Nodes[i])) cancel() @@ -68,12 +68,12 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } else { log.Debug(logs.ReplicatorObjectSuccessfullyReplicated) - task.quantity-- + task.NumCopies-- - res.SubmitSuccessfulReplication(task.nodes[i]) + res.SubmitSuccessfulReplication(task.Nodes[i]) p.metrics.IncProcessedObjects() - p.metrics.AddPayloadSize(int64(task.obj.PayloadSize())) + p.metrics.AddPayloadSize(int64(task.Obj.PayloadSize())) } } } diff --git a/pkg/services/replicator/task.go b/pkg/services/replicator/task.go index ec1b55788..d2b5b2506 100644 --- a/pkg/services/replicator/task.go +++ b/pkg/services/replicator/task.go @@ -8,31 +8,12 @@ import ( // Task represents group of Replicator task parameters. type Task struct { - quantity uint32 - - addr oid.Address - - obj *objectSDK.Object - - nodes []netmap.NodeInfo -} - -// SetCopiesNumber sets number of copies to replicate. -func (t *Task) SetCopiesNumber(v uint32) { - t.quantity = v -} - -// SetObjectAddress sets address of local object. -func (t *Task) SetObjectAddress(v oid.Address) { - t.addr = v -} - -// SetObject sets object to avoid fetching it from the local storage. -func (t *Task) SetObject(obj *objectSDK.Object) { - t.obj = obj -} - -// SetNodes sets a list of potential object holders. -func (t *Task) SetNodes(v []netmap.NodeInfo) { - t.nodes = v + // NumCopies is the number of copies to replicate. + NumCopies uint32 + // Addr is the address of the local object. + Addr oid.Address + // Obj is the object to avoid fetching it from the local storage. + Obj *objectSDK.Object + // Nodes is a list of potential object holders. + Nodes []netmap.NodeInfo } -- 2.45.2 From 3223402c9050db37ff421560e855cd01e967a989 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 30 Jun 2023 10:08:36 +0300 Subject: [PATCH 170/233] [#92] Embed policer's objectsInWork mutex Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 4be13e0d8..d49ad4632 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -11,28 +11,28 @@ import ( ) type objectsInWork struct { - m sync.RWMutex + sync.RWMutex objs map[oid.Address]struct{} } func (oiw *objectsInWork) inWork(addr oid.Address) bool { - oiw.m.RLock() + oiw.RLock() _, ok := oiw.objs[addr] - oiw.m.RUnlock() + oiw.RUnlock() return ok } func (oiw *objectsInWork) remove(addr oid.Address) { - oiw.m.Lock() + oiw.Lock() delete(oiw.objs, addr) - oiw.m.Unlock() + oiw.Unlock() } func (oiw *objectsInWork) add(addr oid.Address) { - oiw.m.Lock() + oiw.Lock() oiw.objs[addr] = struct{}{} - oiw.m.Unlock() + oiw.Unlock() } // Policer represents the utility that verifies -- 2.45.2 From 26acf5689e19eeb15248a12eb5e4a3a8477e6fda Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 30 Jun 2023 10:12:17 +0300 Subject: [PATCH 171/233] [#92] Ensure policer objects cannot be worked on concurrently Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer.go | 4 +++- pkg/services/policer/process.go | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index d49ad4632..a68b194d4 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -29,10 +29,12 @@ func (oiw *objectsInWork) remove(addr oid.Address) { oiw.Unlock() } -func (oiw *objectsInWork) add(addr oid.Address) { +func (oiw *objectsInWork) add(addr oid.Address) bool { oiw.Lock() + _, exists := oiw.objs[addr] oiw.objs[addr] = struct{}{} oiw.Unlock() + return !exists } // Policer represents the utility that verifies diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 6bd38f618..39b61c8a0 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -51,12 +51,11 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { return } - p.objsInWork.add(addr.Address) - - p.processObject(ctx, addr) - - p.cache.Add(addr.Address, time.Now()) - p.objsInWork.remove(addr.Address) + if p.objsInWork.add(addr.Address) { + p.processObject(ctx, addr) + p.cache.Add(addr.Address, time.Now()) + p.objsInWork.remove(addr.Address) + } }) if err != nil { p.log.Warn(logs.PolicerPoolSubmission, zap.Error(err)) -- 2.45.2 From 8d16d95376a7631ad8078cb988a056b8719a0291 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Jun 2023 10:26:38 +0300 Subject: [PATCH 172/233] [#484] morph: Add expired tx logging Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/listener.go | 5 ++++- pkg/morph/event/notary_preparator.go | 18 ++++++++++++++---- pkg/morph/event/notary_preparator_test.go | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 405165702..ca5031415 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -365,11 +365,14 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // prepare the notary event notaryEvent, err := l.notaryEventsPreparator.Prepare(nr.NotaryRequest) if err != nil { + var expErr *ExpiredTXError switch { case errors.Is(err, ErrTXAlreadyHandled): - case errors.Is(err, ErrMainTXExpired): + case errors.As(err, &expErr): l.log.Warn(logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), + zap.Uint32("current_block_height", expErr.CurrentBlockHeight), + zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: l.log.Warn(logs.EventCouldNotPrepareAndValidateNotaryEvent, diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index a8b7376fa..298a6d574 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -39,11 +39,18 @@ var ( // ErrTXAlreadyHandled is returned if received TX has already been signed. ErrTXAlreadyHandled = errors.New("received main tx has already been handled") - - // ErrMainTXExpired is returned if received fallback TX is already valid. - ErrMainTXExpired = errors.New("received main tx has expired") ) +// ExpiredTXError is returned if received fallback TX is already valid. +type ExpiredTXError struct { + CurrentBlockHeight uint32 + FallbackTXNotValidBeforeHeight uint32 +} + +func (e *ExpiredTXError) Error() string { + return "received main tx has expired" +} + // BlockCounter must return block count of the network // from which notary requests are received. type BlockCounter interface { @@ -308,7 +315,10 @@ func (p Preparator) validateExpiration(fbTX *transaction.Transaction) error { } if currBlock >= nvb.Height { - return ErrMainTXExpired + return &ExpiredTXError{ + CurrentBlockHeight: currBlock, + FallbackTXNotValidBeforeHeight: nvb.Height, + } } return nil diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index b2e46890b..8da9d868a 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -331,7 +331,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, }, }, - expErr: ErrMainTXExpired, + expErr: &ExpiredTXError{}, }, { name: "incorrect invoker TX Alphabet witness", -- 2.45.2 From 90e9a85acca029521ed3ab23c20f5d0e977b80c8 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 29 Jun 2023 09:44:11 +0300 Subject: [PATCH 173/233] [#483] Update dependencies Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/internal/client/client.go | 6 +- go.mod | 69 +++++++++--------- go.sum | Bin 99899 -> 99007 bytes pkg/core/client/client.go | 2 +- pkg/network/cache/multi.go | 2 +- pkg/services/object/internal/client/client.go | 6 +- 6 files changed, 44 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 0e0aadbb3..d0bbc9cff 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -396,7 +396,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { return nil, fmt.Errorf("init object writing: %w", err) } - if wrt.WriteHeader(*prm.hdr) { + if wrt.WriteHeader(ctx, *prm.hdr) { if prm.headerCallback != nil { prm.headerCallback(prm.hdr) } @@ -426,7 +426,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { for { n, err = prm.rdr.Read(buf) if n > 0 { - if !wrt.WritePayloadChunk(buf[:n]) { + if !wrt.WritePayloadChunk(ctx, buf[:n]) { break } @@ -442,7 +442,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { } } - cliRes, err := wrt.Close() + cliRes, err := wrt.Close(ctx) if err != nil { // here err already carries both status and client errors return nil, fmt.Errorf("client failure: %w", err) } diff --git a/go.mod b/go.mod index aca1d2d6f..4be07aef7 100644 --- a/go.mod +++ b/go.mod @@ -4,40 +4,40 @@ go 1.19 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb - git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb + git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.2 - github.com/klauspost/compress v1.16.5 + github.com/hashicorp/golang-lru/v2 v2.0.4 + github.com/klauspost/compress v1.16.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 - github.com/nats-io/nats.go v1.25.0 + github.com/nats-io/nats.go v1.27.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.7.4 + github.com/panjf2000/ants/v2 v2.7.5 github.com/paulmach/orb v0.9.2 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.3 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sync v0.2.0 - golang.org/x/term v0.8.0 - google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/sync v0.3.0 + golang.org/x/term v0.9.0 + google.golang.org/grpc v1.56.1 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,12 +45,13 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.8.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.9.1 // indirect + github.com/consensys/gnark-crypto v0.11.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -61,36 +62,36 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.7 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.43.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -99,22 +100,24 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/urfave/cli v1.22.13 // indirect - go.mongodb.org/mongo-driver v1.11.6 // indirect + github.com/urfave/cli v1.22.14 // indirect + go.mongodb.org/mongo-driver v1.12.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/proto/otlp v0.20.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index dfa8632aa1b0773da300effd7988e3d426c725c0..03e60464114b5feabac5869ee9251cc7f87ac204 100644 GIT binary patch delta 5167 zcmb7Id8{k>%9*;zPA)7yc*FgYg*OF)|%tyeNDB> zEX0Nj9Px^+)`#eRp-!_wKblcxPcPA^dg-wzYch2IjmKVWeArKuBV%PBnad~&la^UO zJL%&&cuX7v=fqvQIor%7ogNp3PIzV1?q(CRX|Yp{>?F$pW~1T|f5jHf9`SYj_Uj0b!Csd6TMO$18=RKZhf%&7vM(qE5M5|2;P#dY|5Up+%lV`5~B-u zYA#ti*e?0rdelUe$ksL+rUf#j1#S9$ig#I&U{yyf@ze8~E3E?7vEW?Y{k;z4oKPMN8nmE>SBQK#5XcE|rl! zX#3G<8JBUJZS@yxUk72h&!kx*R#%<(;pjo^!sHn>H%LU@LYC~TfCMKwIGcrRX_{+V z>$bb}Qb)o`Yg=ORc&_$c@9bo77$8&6wt0Z3%h|w`voJv@IB`mz2Io@pLdG+(& zQ~O_<|5*b-&W{Yv_A=d?`!$fMRvrR1_vUm_$!djEtrMk8@#5h{h<84wsvpv}lxAZ*#crF_Y3}c1%yWw^; z*sWGPzovT*)}8hukCS>{N4C6@M@YLA#p?Gy`VJgCD7IW;6fgQ?do}L}YiA_1R89j$ zG9G8LsAY=@T5yZhREGni0)F&H_3X#4?YWO#y9V=Qrjl&1g?b)24b!LlpX(v`&2uLni^annZba-`Q zc_bj(2J7=lJ`^kEnMe2ceB!H@ApH0uYbn!K#*#Gn$#5l=g3>Qt97PdnCU#hyH;qxy z8Mb?>HH@Na{p|JXz0cl5*3g6Kv$Z)LFC}KlZbzCas<pi9)=5*_+3gHVo0x&g+Q8D6i?T*eX7eVp|L^CX zxgC1##oq-juOL0zjM-k4o0gBXwdPtC-C>p)DF$IqH|d#p8jgtS#m~O2lAga-{o0>j zZR$z3jPsU2Mj`^F96lmIG!+SQ*=4f1&yl^dzcHu+5qRqYcmHSauRid?mA&|x7jA=) zTc*=Q+p)rJ{dSuy5^*=D+VE(QuN7X}E&~ff1+zaiV_TA3t>$<;S;G!8j$Epr>!t!a ztyMM;b+$xWn~q6RYj6?`M|}w|fXRfM)Rn28`23Um=U@EPr8n#zWvF@J2I3M!$xN3` zhopwk03!^%V}T>il$nezyP0b4f^+b9TK#G}2q8o2P}-KrMrw;Rr>e>Xdbgm)*0hv+ zQ*P6owcD+(xWD=N>$mSe^qa3H}YV03wbrY_?-fiiHa+ge(GKZ_INYd)0JwJq^*FvUNNa-8sXFqFCdg z=6K?A4ci}m$-K0G<#+$%HK%q=(~DJ~Hwk6&PRmdEX3xqY5O_olFU^#)1xxd(e_Grfj;{HP~zyDJ8zCXPC1UUu| zEfCv{1r9?oxoE<9eLvELGBMzwnD1h0Ri+VH?9kL;(pE>SZod3rb^jkdxc}^z(@XWW zb1^0vask7OaMg;HjGM`dpew{6b!CMmI>u%$jJ$5TQB6ShnV-D*Bw3>l3P9;W;mNBm zH;#IEpf9_@gwL!A1n@9{hraBFc;tuiFyw*#=Wf1qCkz3%qP7Kz0xi8wu|re!2N~Px z@P0S$*jgL69j8Y-G&szho#ay0C;#$#^&4NkdUB3Ehz%?AMJZE4yPuP5D_AWcX2#01 zg(i3HvDZ=+w@U~m+?=eo`^MM4b=SinKtdphq9A}D0R(Bf28!XeBkzCa@0tx5d`+3a zZzfQYlFfiFha*?=3ZxqcOxqG;YTg@a^K!SHkfp_jVr9JYwv%(*fvN~`1}@In4(R4h zZ!2x}WYk-dOd@wBY=h~!rV_d4Y6x08`9D0i_x`?h8&kU*1lIEcc|_tS=wKLVqPBbP z#A6CFx|1+W`b)jun}j?hy&9Z-zG|JCN9&jTwgL z<`OO`niCNN0F+#xmkD(yrY`0X#wZEe>}G0Pd-k7x;;s+Y4t-=h`7v-@Us(`i#d&N#oZ2Kf86fsztp~n1+19^VJJ;zUGltmev2IL|>eJu4 z{)QKtV+p`|n{=gGRtsO^;H>Q+Q)@HQn=sV_Ig~7?^(G?l^Ktd*e}8oU)W7cTKTB_G zT)kAu|9NEq0XPKJ{e@us$N*8oh6uavcRf|SKC_?9R8KNH(>%cXJAO44vU!u~_-l}F z=Z-e+x$7q3kik;qNUM(UZDp(vrMceccI{Lmhja|q;|c09N@v|d1V~kyMht8g1r!a= zm0kZ%mxhFvUu?AXyg!+RPBf%yRED?95sUEso!pOiL!_TbHW(?VU%%3L{OqYKjaM$8 zCigU!XX3q$?Do?ieoN!Z!*3*rqDKUQxh4tYpkpGZe{yd_JpG+(4Y^*bE?#Ok?Pznc zz{*j}NP6Q~-V$v=NGvU7%T*#!tHlOYd!&qE%<13U*SJB{xPvlv$9P+vmQXsFHsMW1 zb>v{)DOy4^W2hYTQU#b0uuiIr=IKWsYCLxK;#(Wjb3N#+MbovC`_*oJ@KAeIWEJ3$mR}5o6qsx9+0)D4`jdFPz=xLBlB{3;+ z*Q5sZDuYZ#qG(M!Q7a0GwkVio(_qdhr14)jPU<`H@JjRsEY};a;sD16>&@JyQEC&S zoj7UsLSQ2GT0Y4^%~he9)+zYD#=~bnAT&Pu@EPVbp1aJRn^!#+1GOn1IY_;6fQ|t; z1a$6%Q`=pmJqt89Y-g(JgV>J|3gpb@Fq{|@XaUkfYi)|+i0z+(PdDB<_-z~J2FiJH9=aepIY%Fe>62z3&NsYk?AVPwrnamt4KqAQ zv8c}pe&M<=5c#T4#*h9)eKHH pQ!nTj(4dXsQPA$m+Yas}#On7aaaU3Uc5~CG-zr=~- z*l}#bScQaOqlk6h7HuD3fPiV*C?%*bS|!>}j3UOgjmrdl7<3;1A%@lu{emWh-}kg4 zt%{=LOZ+^)=kj}Qe&LrdJ^$3DKk8gzafCuqgv8Jjlu>EZ!VCnnw1*t@<`5pOwSIrY zgzQw26b5b>TS{LqQ>`{0@BvAS!+bEAt#zhXJbCwH#o3knj^AF}{`*%ZM|B)U2pYjo zRF$?YQ`1Qtqt9^+EWsy8!W()x8_iHyL?YeTjyLl~%}U}*J)h4T2%AK6v3h~?$i4Ty z=k0Xuzj^P!AKiwXqWeF6&of7Se`)ru@&`^+M~$3^67&>1#b7%)T?EECZhL5~{GO+D zhl^TRO>K~f9yVG?jfFMOw-tSqS6P4JdyYQdD_8&d5_~Q+V6gcnlEOMVm9d^TpRgk_ z@j7zYHRBxZ=c!Vo7VQr2ZRjF+;NgONpk4gdgLmzj2Y(JKp1FSSUcY{F>G5-8&{O2V znCW^LN~j!$6RWj$R@E5L>P8BX$*4O-my%@TwqsR8akA8lZ{K)i|G69ZI}7FX#{?8% za0J0orpzEh0D{&KgJBHfKsUKcIw`R*!BkvY9HS73VN4=pi%sB-Vh{M{bQE+=NpHAZ z@zv809^)v6VJHsh6HC+8QoX6urcu0h+9>|=G%UYYj~NU@FoeM96ATz8g;BJn)5PnK z9A8Bd6h$c<;NV2pFxpUY0-zSUf9>=x2xCss^GGh%8vw$tG4XL0!GTi^9Mtm|gL35M zYz9`AM#hUE87?@k_!xY(=pyfCO3?$qRID-Ji3-TJO)^p~)>N9T*pghgrFnU)DUAgz zvJq0=jEVhcknde8en`HvGz~*31jkVVIUz|}Loo~B3{za8%=7=tcmfnd$?^g8gv2#q z;5xub?J!r4@5=Csz1X zlx}w7YH(HTcab5F8%=E%CMk=LX9mrZ5gKd;l*_JQgrCU79%Y}nhT!KR3mDLCl-L@Y z8uQI!24)&BA@CBbY-{OwWm|aLwz~ExOOkiuHmfS(3 z9$4fcstkv=+u|}F_dGH6ajfX84-|bpWvDlq?bwsrkn)3dzbbPbcF=PPB@z?Hk&_8W zR&)m-3!D)xtQ(aETq`x)S-hykPm>iGGH&kqM!4AoRi}~bx)#m*JQ+9Y z!0QdO6wlMw@h_z5Qu@FNYGnvpc~YyjTW&p78NV9kVZGDStPwZtP(ja%S9E3El9A#? zJ-m*ZynP=kG+ox&2&I#)7dh&J{JUOtVdtuK0YF4{0$5YX?zKvKg#l!}Bw)_;B>z zLRepcK(SxXiW@(B z=ZLDV1)d~qeLO%IZ5&pzMm<)VI;y%`wOUPMeM!&e%5<5`OT-UShh;CxauC|6VF5Cy zZG1KMco_FtFt(Qdx-+HYtZCcCGITbx;?DTy{_mp3od|M{nxL^-AsA;qQL;qsZr6hv zZTe;^g$r_qs6F1R3(Uj3Dai_?$4i#luvlQih*vlb1iiaZ!+QWh`~u^sct1cYmjNi>X)Ni#7vO}&)A zwtwp5|9FIf(+iJd0B`GJOLrw|iuX43pzji%+fk?Lq(_JGoZk?B-cpiTn=M{_>bUsI zQ+G0@=m9Mo*rdvHPEN^m4_US8p`OwmW$N2xKg@yLsw@ZdF1_p+iM@aIsW078>^^

y5S}FU`p8V|P zcike%q7(KSaZjHsew^_HpjRP{axKAf8xCvdQxL(v<88$W*DHcAJOUB>Pk-SF2%*kX zz1S@EkvNQ(T`7owv68!Gz_6xl5L&PA2&U*UbcC}lvl#Rj{>1H< z{GY7d!5|EJAVcX@qrG)hOsEVR6?QC)%=!Mw-}%Hl%3xn4nvSQd(W0YeZQy4{-V}p$ z$aRnoL3)OR$C$2yP8+2xMb6n6FTs zuBC{a%^k3bO6d0$ANlg-n?xyl;I&Xs)<7R{4nBkJjKX2bkmEYDrqgttZad3b1@mio zEnSy1d-|D&_mw~V+)+96E@-yR>+OU~dSIAN!dbGlWuD49Gn&tFvAZOg?jT;-y;;X{ zO3$BpxSV%~$dy%%$hU#Rj`HSm;jc_GGcbRsR}h#WxtSu>VBHyS)3GuuzWD6@`{3FC zzWsVRcTSNL3_cNZfETd`A zqDz-l((QA@RqLVp~9YLZPgs_wdu5xe8!@vmNcOD$MUeN^kDiR0wMj;mz!vLc!j zGN%1X*Brr{QB_hAuiYQ;)#8)SKk&!T|Jv;kR$wnE7X=}Sc#N&KO-k>vS{*gQxJIr5 z4_~@ln6Z~KQ4$ga6JW=)PyN~T;^Qy0if_Gm-~LlCUOu{zZ%2*Fz#T2As7qRYugfLE zVio3SxL%347?EDT%XND}klDmO``Z0Sw`S`$;yHfWqRUYtL?#?oZCuf=JPYLTMnwyJ%gnK9o)@fxRskUdjS2eFY+KAyL1 zl4=DrOhBtkUZCg!zs}P+pLMpB)X`8o@Aq?NyZ`xr`;E&OeJd^ec(z#$O<_F}>aARY z9l18ggtpQroT)CW6Ld}tQfggpYjhF5c5VNSSF?NgGSX$4m#IBbfyywthEt|>xDu=Y zu|*65P-GpMwmpW_hB4{uOH_;-vk6a5`&!(tOLd_f&ewl*tz3o)3`c1K;PeTw%9F$( z%iBRcBwoK$jhfT?#HYt|of^9J5zKUln^ens6hSd?RraS%Dc$tyW!?Pf+F1>P-hEV_ zZ@1i>VlX_;g~oQai0k~48UKqdCMsJ~mt?aVgx1it7mH29{)3;s9lHIPKyef&NCH0rCW!)o8Kh3p zyWhM7$-58V0WIHK-Yp1G29ZF)iGiC2qG=dG>bD*_`|=&o<{i6VzXFLEcFO^AELBFt zc3T}THp9`(=z6m@2SyDHn`9uc`7tP0mp-4?*4*x%d!Vc3eTwXm%TViVbq@rNk8!-* z`w$YPP6$gg%2UhKb%NNv@*p(a{q6^$e>eu^Emd~!G64)lSsI0r7=lo{ryqsnvtM`= z`oYnA5CWkI3qG^tT^^%IoWvp=@o@N@zhdK`GYF7^2)q$~2z6@{BqF!Y_?bANNP+J3D#uXT(B&h zkLlerDD=SDYY6l&SI=&6&@a9BzA{k+N}|XKOy`9D z>GD&u?1V;j6sIViA+*Ew1=MrC)$NefXvkqTM(nR#H)}a6QuR8C!i{aI7o$c=CugUA z1bVE6ms~_js!{}VVyG%b03D%d^J0_guD0{M-DR6pjZsn?r6+k`r3|lAX{(@-$4)O( zW&?{|V%6PCPe3<%|DUph^enq9>~bm}E-`Jeoz`sHLTzm3JLDSl1W;cn$tG9oc2wWJ zw1BRiJ?21nU1IPzABuv+j+|!Ll~4~Ce4ATRvPMk!UeD&=L)mg1NhWeS%DIgUr{cr%$&aq5Nk6ee~}C1>U@o)c^nh diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index bd186006e..422ca1a1a 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -12,7 +12,7 @@ import ( // node's client. type Client interface { ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) - ObjectPutInit(context.Context, client.PrmObjectPutInit) (*client.ObjectWriter, error) + ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) ObjectGetInit(context.Context, client.PrmObjectGet) (*client.ObjectReader, error) ObjectHead(context.Context, client.PrmObjectHead) (*client.ResObjectHead, error) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index ba81df16c..b7f7dc20f 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -219,7 +219,7 @@ func (s *singleClient) invalidate() { s.Unlock() } -func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res *client.ObjectWriter, err error) { +func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res client.ObjectWriter, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ObjectPutInit(ctx, p) return err diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 17bf6a40f..a54f6cd01 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -429,11 +429,11 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { return nil, fmt.Errorf("init object writing on client: %w", err) } - if w.WriteHeader(*prm.obj) { - w.WritePayloadChunk(prm.obj.Payload()) + if w.WriteHeader(ctx, *prm.obj) { + w.WritePayloadChunk(ctx, prm.obj.Payload()) } - cliRes, err := w.Close() + cliRes, err := w.Close(ctx) if err == nil { err = apistatus.ErrFromStatus(cliRes.Status()) } else { -- 2.45.2 From 4c248d573eb7951abca58e6ef89556f6d9f0dc1a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 29 Jun 2023 09:48:32 +0300 Subject: [PATCH 174/233] [#483] cli: Allow to split object on the client side Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/internal/client/client.go | 64 +++++++++++++++++------ cmd/frostfs-cli/modules/object/put.go | 36 ++++++++----- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index d0bbc9cff..9545c60cb 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -336,6 +336,8 @@ type PutObjectPrm struct { rdr io.Reader headerCallback func(*object.Object) + + prepareLocally bool } // SetHeader sets object header. @@ -360,6 +362,41 @@ func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) { x.copyNum = copiesNumbers } +// PrepareLocally generate object header on the client side. +// For big object - split locally too. +func (x *PutObjectPrm) PrepareLocally() { + x.prepareLocally = true +} + +func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) { + var putPrm client.PrmObjectPutInit + if !x.prepareLocally && x.sessionToken != nil { + putPrm.WithinSession(*x.sessionToken) + } + + if x.bearerToken != nil { + putPrm.WithBearerToken(*x.bearerToken) + } + + if x.local { + putPrm.MarkLocal() + } + + putPrm.WithXHeaders(x.xHeaders...) + putPrm.SetCopiesNumberByVectors(x.copyNum) + + if x.prepareLocally { + res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{}) + if err != nil { + return client.PrmObjectPutInit{}, err + } + putPrm.WithObjectMaxSize(res.Info().MaxObjectSize()) + putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch())) + putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled()) + } + return putPrm, nil +} + // PutObjectRes groups the resulting values of PutObject operation. type PutObjectRes struct { id oid.ID @@ -370,28 +407,21 @@ func (x PutObjectRes) ID() oid.ID { return x.id } +type epochSource uint64 + +func (s epochSource) CurrentEpoch() uint64 { + return uint64(s) +} + // PutObject saves the object in FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { - var putPrm client.PrmObjectPutInit - - if prm.sessionToken != nil { - putPrm.WithinSession(*prm.sessionToken) + sdkPrm, err := prm.convertToSDKPrm(ctx) + if err != nil { + return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err) } - - if prm.bearerToken != nil { - putPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.local { - putPrm.MarkLocal() - } - - putPrm.WithXHeaders(prm.xHeaders...) - putPrm.SetCopiesNumberByVectors(prm.copyNum) - - wrt, err := prm.cli.ObjectPutInit(ctx, putPrm) + wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm) if err != nil { return nil, fmt.Errorf("init object writing: %w", err) } diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index af9e9fd0b..19edbb975 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -23,9 +23,10 @@ import ( ) const ( - noProgressFlag = "no-progress" - notificationFlag = "notify" - copiesNumberFlag = "copies-number" + noProgressFlag = "no-progress" + notificationFlag = "notify" + copiesNumberFlag = "copies-number" + prepareLocallyFlag = "prepare-locally" ) var putExpiredOn uint64 @@ -54,6 +55,7 @@ func initObjectPutCmd() { flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute") flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object") flags.Bool(noProgressFlag, false, "Do not show progress bar") + flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)") flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") @@ -102,7 +104,11 @@ func putObject(cmd *cobra.Command, _ []string) { } var prm internalclient.PutObjectPrm - ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally { + prm.PrepareLocally() + } else { + ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + } Prepare(cmd, &prm) prm.SetHeader(obj) @@ -121,15 +127,7 @@ func putObject(cmd *cobra.Command, _ []string) { copyNum, err := cmd.Flags().GetString(copiesNumberFlag) commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) - if len(copyNum) > 0 { - var cn []uint32 - for _, num := range strings.Split(copyNum, ",") { - val, err := strconv.ParseUint(num, 10, 32) - commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) - cn = append(cn, uint32(val)) - } - prm.SetCopiesNumberByVectors(cn) - } + prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum)) res, err := internalclient.PutObject(cmd.Context(), prm) if p != nil { @@ -141,6 +139,18 @@ func putObject(cmd *cobra.Command, _ []string) { cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr) } +func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 { + var cn []uint32 + if len(copyNum) > 0 { + for _, num := range strings.Split(copyNum, ",") { + val, err := strconv.ParseUint(num, 10, 32) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + cn = append(cn, uint32(val)) + } + } + return cn +} + func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) -- 2.45.2 From b520a3049e6f4ffeb12fa0f7a4cf7c5d95579bf0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 4 Jul 2023 18:41:25 +0300 Subject: [PATCH 175/233] [#483] cli: Fix `object put` cmd with flag `--prepare-locally` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/put.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 19edbb975..c5b4e6e06 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -105,6 +105,7 @@ func putObject(cmd *cobra.Command, _ []string) { var prm internalclient.PutObjectPrm if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally { + prm.SetClient(internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)) prm.PrepareLocally() } else { ReadOrOpenSession(cmd, &prm, pk, cnr, nil) -- 2.45.2 From c6df6c84ae3ef2e9f67aa79382405a2aa58eb025 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 5 Jul 2023 19:49:58 +0300 Subject: [PATCH 176/233] [#473] Do not modify traverse plan when `[0]` copies number vector is provided Signed-off-by: Alex Vanin --- pkg/services/object_manager/placement/traverser.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 47f79a627..c59146e2b 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -97,10 +97,16 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } else { rem = defaultCopiesVector(cfg.policy) + // compatibleZeroVector is a bool flag which is set when cfg.copyNumbers + // is [0]. In this case we should not modify `rem` slice unless track + // copies are ignored, because [0] means that all copies should be + // stored before returning OK to the client. + compatibleZeroVector := len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] == 0 + for i := range rem { if !cfg.trackCopies { rem[i] = -1 - } else if len(cfg.copyNumbers) > i { + } else if len(cfg.copyNumbers) > i && !compatibleZeroVector { rem[i] = int(cfg.copyNumbers[i]) } } -- 2.45.2 From f354b8a270b3cb5486df4927f4c461715a999b56 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 5 Jul 2023 20:12:55 +0300 Subject: [PATCH 177/233] [#473] Add tests for rem values of traverser Signed-off-by: Alex Vanin --- .../placement/traverser_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 66fd8afe0..1b307da6f 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -208,3 +208,55 @@ func TestTraverserObjectScenarios(t *testing.T) { require.True(t, tr.Success()) }) } + +func TestTraverserRemValues(t *testing.T) { + selectors := []int{3, 4, 5} + replicas := []int{2, 3, 4} + + nodes, cnr := testPlacement(t, selectors, replicas) + nodesCopy := copyVectors(nodes) + + testCases := [...]struct { + name string + copyNumbers []uint32 + expectedRem []int + }{ + { + name: "zero copy numbers", + copyNumbers: []uint32{}, + expectedRem: replicas, + }, + { + name: "compatible zero copy numbers", + copyNumbers: []uint32{0}, + expectedRem: replicas, + }, + { + name: "copy numbers for all replicas", + copyNumbers: []uint32{1, 1, 1}, + expectedRem: []int{1, 1, 1}, + }, + { + name: "single copy numbers for multiple replicas", + copyNumbers: []uint32{1}, + expectedRem: []int{1}, // may be a bit unexpected + }, + { + name: "multiple copy numbers for multiple replicas", + copyNumbers: []uint32{1, 1}, + expectedRem: []int{1, 1, 4}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{vectors: nodesCopy}), + WithCopyNumbers(testCase.copyNumbers), + ) + require.NoError(t, err) + require.Equal(t, testCase.expectedRem, tr.rem) + }) + } +} -- 2.45.2 From 6eefe9747e28c709d1b1ab6248a528581bd76285 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Jul 2023 17:07:52 +0300 Subject: [PATCH 178/233] treesvc: Do not provide credentials unless TLS is used 4f413fe86e was perfect, except it did the opposite of what we needed. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index ef0c4b465..f50aa0b0d 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -99,7 +99,7 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, ), } - if netAddr.IsTLSEnabled() { + if !netAddr.IsTLSEnabled() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } -- 2.45.2 From 4bbe9cc9361b1c8be06b8ba1f6706048549eb56b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 6 Jul 2023 15:35:23 +0300 Subject: [PATCH 179/233] [#496] .golangci.yml: Add importas linter Add importas linter Signed-off-by: Alexander Chuprov --- .golangci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 4a3fec6e3..643390075 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -31,6 +31,12 @@ linters-settings: statements: 60 # default 40 gocognit: min-complexity: 40 # default 30 + importas: + no-unaliased: true + no-extra-aliases: false + alias: + pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object + alias: objectSDK linters: enable: @@ -62,5 +68,6 @@ linters: - funlen - gocognit - contextcheck + - importas disable-all: true fast: false -- 2.45.2 From 033eaf77e102e66bd4f144197bfa22f8bfc847d1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 6 Jul 2023 15:36:41 +0300 Subject: [PATCH 180/233] [#496] node: Fix linter importas Standardize the alias of the import frostfs-sdk-go/object as objectSDK. Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/internal/client/client.go | 38 +++++++-------- .../modules/container/list_objects.go | 4 +- cmd/frostfs-cli/modules/object/get.go | 4 +- cmd/frostfs-cli/modules/object/head.go | 12 ++--- cmd/frostfs-cli/modules/object/put.go | 32 ++++++------- cmd/frostfs-cli/modules/object/range.go | 14 +++--- cmd/frostfs-cli/modules/object/search.go | 22 ++++----- cmd/frostfs-cli/modules/object/util.go | 20 ++++---- cmd/frostfs-cli/modules/tree/add_by_path.go | 4 +- cmd/frostfs-cli/modules/tree/get_by_path.go | 4 +- .../internal/blobovnicza/inspect.go | 4 +- cmd/frostfs-lens/internal/meta/inspect.go | 4 +- cmd/frostfs-lens/internal/printers.go | 4 +- .../internal/writecache/inspect.go | 4 +- pkg/core/object/address.go | 4 +- pkg/core/object/fmt.go | 34 +++++++------- pkg/core/object/fmt_test.go | 40 ++++++++-------- pkg/core/object/object.go | 4 +- .../engine/engine_test.go | 6 +-- pkg/local_object_storage/engine/head_test.go | 6 +-- pkg/local_object_storage/engine/lock_test.go | 16 +++---- pkg/local_object_storage/engine/select.go | 8 ++-- pkg/local_object_storage/engine/tree_test.go | 6 +-- .../internal/testutil/generators.go | 14 +++--- .../internal/testutil/object.go | 16 +++---- .../metabase/counter_test.go | 10 ++-- pkg/local_object_storage/metabase/db.go | 16 +++---- .../metabase/expired_test.go | 4 +- pkg/local_object_storage/metabase/inhume.go | 6 +-- .../metabase/iterators.go | 7 +-- .../metabase/iterators_test.go | 18 ++++---- pkg/local_object_storage/metabase/list.go | 12 ++--- pkg/local_object_storage/metabase/lock.go | 4 +- .../metabase/lock_test.go | 20 ++++---- pkg/local_object_storage/metabase/select.go | 46 +++++++++---------- pkg/local_object_storage/metabase/util.go | 14 +++--- pkg/local_object_storage/shard/gc.go | 6 +-- .../shard/gc_internal_test.go | 4 +- pkg/local_object_storage/shard/list.go | 4 +- pkg/local_object_storage/shard/lock_test.go | 6 +-- .../shard/metrics_test.go | 6 +-- pkg/local_object_storage/shard/put.go | 6 +-- pkg/local_object_storage/shard/range.go | 16 +++---- pkg/local_object_storage/shard/select.go | 6 +-- pkg/local_object_storage/shard/shard_test.go | 4 +- pkg/local_object_storage/util/splitinfo.go | 4 +- .../util/splitinfo_test.go | 16 +++---- pkg/local_object_storage/writecache/flush.go | 12 ++--- .../writecache/flush_test.go | 10 ++-- .../writecache/writecache.go | 12 ++--- pkg/network/cache/multi.go | 6 +-- pkg/services/object/acl/eacl/v2/eacl_test.go | 10 ++-- pkg/services/object/acl/eacl/v2/headers.go | 10 ++-- pkg/services/object/acl/eacl/v2/object.go | 4 +- pkg/services/object/delete/exec.go | 14 +++--- pkg/services/object/delete/local.go | 4 +- pkg/services/object/delete/service.go | 6 +-- pkg/services/object/delete/util.go | 12 ++--- pkg/services/object/get/get.go | 4 +- pkg/services/object/get/v2/get_forwarder.go | 10 ++-- .../object/get/v2/get_range_forwarder.go | 8 ++-- pkg/services/object/get/v2/head_forwarder.go | 10 ++-- pkg/services/object/get/v2/service.go | 8 ++-- pkg/services/object/get/v2/streamer.go | 4 +- pkg/services/object/get/v2/util.go | 26 +++++------ pkg/services/object/get/writer.go | 14 +++--- pkg/services/object/head/remote.go | 4 +- pkg/services/object/internal/client/client.go | 40 ++++++++-------- pkg/services/object/put/local.go | 12 ++--- pkg/services/object/put/prm.go | 6 +-- pkg/services/object/put/remote.go | 10 ++-- pkg/services/object/put/streamer.go | 4 +- pkg/services/object/put/v2/util.go | 4 +- pkg/services/object/search/exec.go | 4 +- pkg/services/object/search/prm.go | 6 +-- pkg/services/object/search/v2/util.go | 4 +- pkg/services/object/util/chain.go | 30 ++++++------ .../object_manager/tombstone/checker.go | 6 +-- pkg/services/policer/check.go | 4 +- pkg/services/policer/policer_test.go | 10 ++-- 80 files changed, 444 insertions(+), 443 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 9545c60cb..babc0ef38 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -13,7 +13,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -331,17 +331,17 @@ type PutObjectPrm struct { copyNum []uint32 - hdr *object.Object + hdr *objectSDK.Object rdr io.Reader - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) prepareLocally bool } // SetHeader sets object header. -func (x *PutObjectPrm) SetHeader(hdr *object.Object) { +func (x *PutObjectPrm) SetHeader(hdr *objectSDK.Object) { x.hdr = hdr } @@ -352,7 +352,7 @@ func (x *PutObjectPrm) SetPayloadReader(rdr io.Reader) { // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (x *PutObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (x *PutObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { x.headerCallback = f } @@ -532,22 +532,22 @@ type GetObjectPrm struct { objectAddressPrm rawPrm payloadWriterPrm - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) } // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (p *GetObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (p *GetObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { p.headerCallback = f } // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the header of the request object. -func (x GetObjectRes) Header() *object.Object { +func (x GetObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -585,7 +585,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { return nil, fmt.Errorf("init object reading on client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !rdr.ReadHeader(&hdr) { _, err = rdr.Close() @@ -621,11 +621,11 @@ func (x *HeadObjectPrm) SetMainOnlyFlag(v bool) { // HeadObjectRes groups the resulting values of HeadObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -661,7 +661,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) return nil, fmt.Errorf("read object header via client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !res.ReadHeader(&hdr) { return nil, fmt.Errorf("missing header in response") @@ -677,11 +677,11 @@ type SearchObjectsPrm struct { commonObjectPrm containerIDPrm - filters object.SearchFilters + filters objectSDK.SearchFilters } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(filters object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(filters objectSDK.SearchFilters) { x.filters = filters } @@ -754,7 +754,7 @@ type HashPayloadRangesPrm struct { tz bool - rngs []*object.Range + rngs []*objectSDK.Range salt []byte } @@ -765,7 +765,7 @@ func (x *HashPayloadRangesPrm) TZ() { } // SetRanges sets a list of payload ranges to hash. -func (x *HashPayloadRangesPrm) SetRanges(rngs []*object.Range) { +func (x *HashPayloadRangesPrm) SetRanges(rngs []*objectSDK.Range) { x.rngs = rngs } @@ -839,11 +839,11 @@ type PayloadRangePrm struct { rawPrm payloadWriterPrm - rng *object.Range + rng *objectSDK.Range } // SetRange sets payload range to read. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.rng = rng } diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index d00cc3576..1e0aeb4db 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -31,7 +31,7 @@ var listContainerObjectsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { id := parseContainerID(cmd) - filters := new(object.SearchFilters) + filters := new(objectSDK.SearchFilters) filters.AddRootFilter() // search only user created objects cli := internalclient.GetSDKClientByFlag(cmd, key.GetOrGenerate(cmd), commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index c72a26239..3136f086f 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/cheggaaa/pb" "github.com/spf13/cobra" @@ -84,7 +84,7 @@ func getObject(cmd *cobra.Command, _ []string) { p = pb.New64(0) p.Output = cmd.OutOrStdout() prm.SetPayloadWriter(p.NewProxyWriter(payloadWriter)) - prm.SetHeaderCallback(func(o *object.Object) { + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.SetTotal64(int64(o.PayloadSize())) p.Start() }) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 04467744d..db466e588 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -77,7 +77,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", err) } -func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) error { +func saveAndPrintHeader(cmd *cobra.Command, obj *objectSDK.Object, filename string) error { bs, err := marshalHeader(cmd, obj) if err != nil { return fmt.Errorf("could not marshal header: %w", err) @@ -97,7 +97,7 @@ func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) return printHeader(cmd, obj) } -func marshalHeader(cmd *cobra.Command, hdr *object.Object) ([]byte, error) { +func marshalHeader(cmd *cobra.Command, hdr *objectSDK.Object) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -138,7 +138,7 @@ func printContainerID(cmd *cobra.Command, recv func() (cid.ID, bool)) { cmd.Printf("CID: %s\n", strID) } -func printHeader(cmd *cobra.Command, obj *object.Object) error { +func printHeader(cmd *cobra.Command, obj *objectSDK.Object) error { printObjectID(cmd, obj.ID) printContainerID(cmd, obj.ContainerID) cmd.Printf("Owner: %s\n", obj.OwnerID()) @@ -150,7 +150,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { cmd.Println("Attributes:") for _, attr := range obj.Attributes() { - if attr.Key() == object.AttributeTimestamp { + if attr.Key() == objectSDK.AttributeTimestamp { cmd.Printf(" %s=%s (%s)\n", attr.Key(), attr.Value(), @@ -174,7 +174,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { return printSplitHeader(cmd, obj) } -func printSplitHeader(cmd *cobra.Command, obj *object.Object) error { +func printSplitHeader(cmd *cobra.Command, obj *objectSDK.Object) error { if splitID := obj.SplitID(); splitID != nil { cmd.Printf("Split ID: %s\n", splitID) } diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index c5b4e6e06..97bb12dbc 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/cheggaaa/pb" "github.com/spf13/cobra" @@ -81,7 +81,7 @@ func putObject(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("can't open file '%s': %w", filename, err)) } var payloadReader io.Reader = f - obj := object.New() + obj := objectSDK.New() if binary { payloadReader, cnr, ownerID = readFilePayload(filename, cmd) @@ -155,7 +155,7 @@ func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 { func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() // TODO(@acid-ant): #1932 Use streams to marshal/unmarshal payload commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) payloadReader := bytes.NewReader(objTemp.Payload()) @@ -174,19 +174,19 @@ func setFilePayloadReader(cmd *cobra.Command, f *os.File, prm *internalclient.Pu p := pb.New64(fi.Size()) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(f)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func setBinaryPayloadReader(cmd *cobra.Command, obj *object.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { +func setBinaryPayloadReader(cmd *cobra.Command, obj *objectSDK.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { p := pb.New(len(obj.Payload())) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(payloadReader)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { +func getAllObjectAttributes(cmd *cobra.Command) []objectSDK.Attribute { attrs, err := parseObjectAttrs(cmd) commonCmd.ExitOnErr(cmd, "can't parse object attributes: %w", err) @@ -205,7 +205,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { if !expAttrFound { index := len(attrs) - attrs = append(attrs, object.Attribute{}) + attrs = append(attrs, objectSDK.Attribute{}) attrs[index].SetKey(objectV2.SysAttributeExpEpoch) attrs[index].SetValue(expAttrValue) } @@ -213,7 +213,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { return attrs } -func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { +func parseObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { var rawAttrs []string raw := cmd.Flag("attributes").Value.String() @@ -221,7 +221,7 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { rawAttrs = strings.Split(raw, ",") } - attrs := make([]object.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes + attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes for i := range rawAttrs { k, v, found := strings.Cut(rawAttrs[i], "=") if !found { @@ -235,23 +235,23 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { if !disableFilename { filename := filepath.Base(cmd.Flag(fileFlag).Value.String()) index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeFileName) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeFileName) attrs[index].SetValue(filename) } disableTime, _ := cmd.Flags().GetBool("disable-timestamp") if !disableTime { index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeTimestamp) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeTimestamp) attrs[index].SetValue(strconv.FormatInt(time.Now().Unix(), 10)) } return attrs, nil } -func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, error) { +func parseObjectNotifications(cmd *cobra.Command) (*objectSDK.NotificationInfo, error) { const ( separator = ":" useDefaultTopic = "-" @@ -267,7 +267,7 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err return nil, fmt.Errorf("notification must be in the form of: *epoch*%s*topic*, got %s", separator, raw) } - ni := new(object.NotificationInfo) + ni := new(objectSDK.NotificationInfo) epoch, err := strconv.ParseUint(before, 10, 64) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 76425d948..591355b95 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -14,7 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -102,7 +102,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { } func printSplitInfoErr(cmd *cobra.Command, err error) bool { - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError ok := errors.As(err, &errSplitInfo) @@ -114,14 +114,14 @@ func printSplitInfoErr(cmd *cobra.Command, err error) bool { return ok } -func printSplitInfo(cmd *cobra.Command, info *object.SplitInfo) { +func printSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) { bs, err := marshalSplitInfo(cmd, info) commonCmd.ExitOnErr(cmd, "can't marshal split info: %w", err) cmd.Println(string(bs)) } -func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error) { +func marshalSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -146,13 +146,13 @@ func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error } } -func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { +func getRangeList(cmd *cobra.Command) ([]*objectSDK.Range, error) { v := cmd.Flag("range").Value.String() if len(v) == 0 { return nil, nil } vs := strings.Split(v, ",") - rs := make([]*object.Range, len(vs)) + rs := make([]*objectSDK.Range, len(vs)) for i := range vs { before, after, found := strings.Cut(vs[i], rangeSep) if !found { @@ -176,7 +176,7 @@ func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { return nil, fmt.Errorf("invalid '%s' range: uint64 overflow", vs[i]) } - rs[i] = object.NewRange() + rs[i] = objectSDK.NewRange() rs[i].SetOffset(offset) rs[i].SetLength(length) } diff --git a/cmd/frostfs-cli/modules/object/search.go b/cmd/frostfs-cli/modules/object/search.go index b603e5fe8..ca5d78bc9 100644 --- a/cmd/frostfs-cli/modules/object/search.go +++ b/cmd/frostfs-cli/modules/object/search.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -72,18 +72,18 @@ func searchObject(cmd *cobra.Command, _ []string) { } } -var searchUnaryOpVocabulary = map[string]object.SearchMatchType{ - "NOPRESENT": object.MatchNotPresent, +var searchUnaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "NOPRESENT": objectSDK.MatchNotPresent, } -var searchBinaryOpVocabulary = map[string]object.SearchMatchType{ - "EQ": object.MatchStringEqual, - "NE": object.MatchStringNotEqual, - "COMMON_PREFIX": object.MatchCommonPrefix, +var searchBinaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "EQ": objectSDK.MatchStringEqual, + "NE": objectSDK.MatchStringNotEqual, + "COMMON_PREFIX": objectSDK.MatchCommonPrefix, } -func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { - var fs object.SearchFilters +func parseSearchFilters(cmd *cobra.Command) (objectSDK.SearchFilters, error) { + var fs objectSDK.SearchFilters for i := range searchFilters { words := strings.Fields(searchFilters[i]) @@ -97,7 +97,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not read attributes filter from file: %w", err) } - subFs := object.NewSearchFilters() + subFs := objectSDK.NewSearchFilters() if err := subFs.UnmarshalJSON(data); err != nil { return nil, fmt.Errorf("could not unmarshal attributes filter from file: %w", err) @@ -138,7 +138,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not parse object ID: %w", err) } - fs.AddObjectIDFilter(object.MatchStringEqual, id) + fs.AddObjectIDFilter(objectSDK.MatchStringEqual, id) } return fs, nil diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 094b62314..37e9f74e0 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "github.com/spf13/cobra" @@ -87,7 +87,7 @@ func readObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address func readObjectAddressBin(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID, filename string) oid.Address { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) var addr oid.Address @@ -356,7 +356,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, _, err := internal.HeadObject(cmd.Context(), prmHead) - var errSplit *object.SplitInfoError + var errSplit *objectSDK.SplitInfoError switch { default: @@ -381,7 +381,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) } -func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { // collect split chain by the descending ease of operations (ease is evaluated heuristically). // If any approach fails, we don't try the next since we assume that it will fail too. @@ -413,12 +413,12 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl return nil, false } -func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { if idSplit := splitInfo.SplitID(); idSplit != nil { common.PrintVerbose(cmd, "Collecting split members by split ID...") - var query object.SearchFilters - query.AddSplitIDFilter(object.MatchStringEqual, idSplit) + var query objectSDK.SearchFilters + query.AddSplitIDFilter(objectSDK.MatchStringEqual, idSplit) var prm internal.SearchObjectsPrm prm.SetContainerID(cnr) @@ -437,7 +437,7 @@ func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo return nil, false } -func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { +func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { var addrObj oid.Address addrObj.SetContainer(cnr) @@ -482,8 +482,8 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p common.PrintVerbose(cmd, "Looking for a linking object...") - var query object.SearchFilters - query.AddParentIDFilter(object.MatchStringEqual, obj) + var query objectSDK.SearchFilters + query.AddParentIDFilter(objectSDK.MatchStringEqual, obj) var prmSearch internal.SearchObjectsPrm prmSearch.SetClient(cli) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 6b182e1e4..90f4a6364 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -68,7 +68,7 @@ func addByPath(cmd *cobra.Command, _ []string) { req.Body = &tree.AddByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 69a3dc7ea..38ad2bff5 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -66,7 +66,7 @@ func getByPath(cmd *cobra.Command, _ []string) { req.Body = &tree.GetNodeByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), LatestOnly: latestOnly, diff --git a/cmd/frostfs-lens/internal/blobovnicza/inspect.go b/cmd/frostfs-lens/internal/blobovnicza/inspect.go index 13442a4b8..b1a6e3fd2 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/inspect.go +++ b/cmd/frostfs-lens/internal/blobovnicza/inspect.go @@ -3,7 +3,7 @@ package blobovnicza import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -38,7 +38,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data := res.Object() - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data)), ) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index bc7f28a3a..de0f24aeb 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -7,7 +7,7 @@ import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -49,7 +49,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { prm.SetAddress(addr) prm.SetRaw(true) - siErr := new(object.SplitInfoError) + siErr := new(objectSDK.SplitInfoError) res, err := db.Get(cmd.Context(), prm) if errors.As(err, &siErr) { diff --git a/cmd/frostfs-lens/internal/printers.go b/cmd/frostfs-lens/internal/printers.go index a232409d6..dd73a5552 100644 --- a/cmd/frostfs-lens/internal/printers.go +++ b/cmd/frostfs-lens/internal/printers.go @@ -4,14 +4,14 @@ import ( "os" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) // PrintObjectHeader prints passed object's header fields via // the passed cobra command. Does nothing with the payload. -func PrintObjectHeader(cmd *cobra.Command, h object.Object) { +func PrintObjectHeader(cmd *cobra.Command, h objectSDK.Object) { cmd.Println("Version:", h.Version()) cmd.Println("Type:", h.Type()) printContainerID(cmd, h.ContainerID) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 91f12ed0f..7d3c8ab22 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -3,7 +3,7 @@ package writecache import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -27,7 +27,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data, err := writecache.Get(db, []byte(vAddress)) common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data))) common.PrintObjectHeader(cmd, o) diff --git a/pkg/core/object/address.go b/pkg/core/object/address.go index cd5559d9f..12e5c89ce 100644 --- a/pkg/core/object/address.go +++ b/pkg/core/object/address.go @@ -1,7 +1,7 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -9,5 +9,5 @@ import ( // object type. type AddressWithType struct { Address oid.Address - Type object.Type + Type objectSDK.Type } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index ba9868693..abf8f02cb 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -87,7 +87,7 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { // If unprepared is true, only fields set by user are validated. // // Returns nil error if the object has valid structure. -func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unprepared bool) error { +func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, unprepared bool) error { if obj == nil { return errNilObject } @@ -119,7 +119,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return fmt.Errorf("object did not pass expiration check: %w", err) } - if err := object.CheckHeaderVerificationFields(obj); err != nil { + if err := objectSDK.CheckHeaderVerificationFields(obj); err != nil { return fmt.Errorf("(%T) could not validate header fields: %w", v, err) } } @@ -132,7 +132,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return nil } -func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { +func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { // TODO(@cthulhu-rider): #468 use "const" error @@ -178,13 +178,13 @@ func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) // - object.TypeTombstone; // - object.TypeLock. type ContentMeta struct { - typ object.Type + typ objectSDK.Type objs []oid.ID } // Type returns object's type. -func (i ContentMeta) Type() object.Type { +func (i ContentMeta) Type() objectSDK.Type { return i.typ } @@ -197,17 +197,17 @@ func (i ContentMeta) Objects() []oid.ID { } // ValidateContent validates payload content according to the object type. -func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) { +func (v *FormatValidator) ValidateContent(o *objectSDK.Object) (ContentMeta, error) { meta := ContentMeta{ typ: o.Type(), } switch o.Type() { - case object.TypeTombstone: + case objectSDK.TypeTombstone: if err := v.fillAndValidateTombstoneMeta(o, &meta); err != nil { return ContentMeta{}, err } - case object.TypeLock: + case objectSDK.TypeLock: if err := v.fillAndValidateLockMeta(o, &meta); err != nil { return ContentMeta{}, err } @@ -218,7 +218,7 @@ func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) return meta, nil } -func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateLockMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return errors.New("empty payload in lock") } @@ -240,7 +240,7 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return fmt.Errorf("lock object expiration: %d; current: %d", lockExp, currEpoch) } - var lock object.Lock + var lock objectSDK.Lock if err = lock.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("decode lock payload: %w", err) @@ -256,12 +256,12 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return nil } -func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateTombstoneMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return fmt.Errorf("(%T) empty payload in tombstone", v) } - tombstone := object.NewTombstone() + tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("(%T) could not unmarshal tombstone content: %w", v, err) @@ -287,7 +287,7 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *C var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Object) error { +func (v *FormatValidator) checkExpiration(ctx context.Context, obj *objectSDK.Object) error { exp, err := expirationEpochAttribute(obj) if err != nil { if errors.Is(err, errNoExpirationEpoch) { @@ -321,7 +321,7 @@ func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Objec return nil } -func expirationEpochAttribute(obj *object.Object) (uint64, error) { +func expirationEpochAttribute(obj *objectSDK.Object) (uint64, error) { for _, a := range obj.Attributes() { if a.Key() != objectV2.SysAttributeExpEpoch && a.Key() != objectV2.SysAttributeExpEpochNeoFS { continue @@ -338,7 +338,7 @@ var ( errEmptyAttrVal = errors.New("empty attribute value") ) -func (v *FormatValidator) checkAttributes(obj *object.Object) error { +func (v *FormatValidator) checkAttributes(obj *objectSDK.Object) error { as := obj.Attributes() mUnique := make(map[string]struct{}, len(as)) @@ -362,7 +362,7 @@ func (v *FormatValidator) checkAttributes(obj *object.Object) error { var errIncorrectOwner = errors.New("incorrect object owner") -func (v *FormatValidator) checkOwner(obj *object.Object) error { +func (v *FormatValidator) checkOwner(obj *objectSDK.Object) error { if idOwner := obj.OwnerID(); idOwner == nil || len(idOwner.WalletBytes()) == 0 { return errIncorrectOwner } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 2cf5099ba..d04c16709 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -8,7 +8,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" @@ -17,11 +17,11 @@ import ( "github.com/stretchr/testify/require" ) -func blankValidObject(key *ecdsa.PrivateKey) *object.Object { +func blankValidObject(key *ecdsa.PrivateKey) *objectSDK.Object { var idOwner user.ID user.IDFromKey(&idOwner, key.PublicKey) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetOwnerID(&idOwner) @@ -66,20 +66,20 @@ func TestFormatValidator_Validate(t *testing.T) { }) t.Run("nil identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() require.ErrorIs(t, v.Validate(context.Background(), obj, false), errNilID) }) t.Run("nil container identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) require.ErrorIs(t, v.Validate(context.Background(), obj, true), errNilCID) }) t.Run("unsigned object", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetID(oidtest.ID()) @@ -94,12 +94,12 @@ func TestFormatValidator_Validate(t *testing.T) { err := tok.Sign(ownerKey.PrivateKey) require.NoError(t, err) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetSessionToken(tok) obj.SetOwnerID(&idOwner) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) @@ -107,20 +107,20 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("correct w/o session token", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("tombstone content", func(t *testing.T) { - obj := object.New() - obj.SetType(object.TypeTombstone) + obj := objectSDK.New() + obj.SetType(objectSDK.TypeTombstone) obj.SetContainerID(cidtest.ID()) _, err := v.ValidateContent(obj) require.Error(t, err) // no tombstone content - content := object.NewTombstone() + content := objectSDK.NewTombstone() content.SetMembers([]oid.ID{oidtest.ID()}) data, err := content.Marshal() @@ -141,7 +141,7 @@ func TestFormatValidator_Validate(t *testing.T) { _, err = v.ValidateContent(obj) require.Error(t, err) // no expiration epoch in tombstone - var expirationAttribute object.Attribute + var expirationAttribute objectSDK.Attribute expirationAttribute.SetKey(objectV2.SysAttributeExpEpoch) expirationAttribute.SetValue(strconv.Itoa(10)) @@ -163,20 +163,20 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, err) // all good require.EqualValues(t, []oid.ID{id}, contentGot.Objects()) - require.Equal(t, object.TypeTombstone, contentGot.Type()) + require.Equal(t, objectSDK.TypeTombstone, contentGot.Type()) }) t.Run("expiration", func(t *testing.T) { - fn := func(val string) *object.Object { + fn := func(val string) *objectSDK.Object { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(val) obj.SetAttributes(a) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) return obj } @@ -221,11 +221,11 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("duplication", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a1 object.Attribute + var a1 objectSDK.Attribute a1.SetKey("key1") a1.SetValue("val1") - var a2 object.Attribute + var a2 objectSDK.Attribute a2.SetKey("key2") a2.SetValue("val2") @@ -244,7 +244,7 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("empty value", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey("key") obj.SetAttributes(a) diff --git a/pkg/core/object/object.go b/pkg/core/object/object.go index 8fb656acd..9c450966c 100644 --- a/pkg/core/object/object.go +++ b/pkg/core/object/object.go @@ -1,12 +1,12 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) // AddressOf returns the address of the object. -func AddressOf(obj *object.Object) oid.Address { +func AddressOf(obj *objectSDK.Object) oid.Address { var addr oid.Address id, ok := obj.ID() diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 11c0848fe..26de2e5eb 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -18,7 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" @@ -148,7 +148,7 @@ func newStorages(root string, smallSize uint64) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), blobovniczatree.WithPermissions(0700)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, @@ -176,7 +176,7 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes return []blobstor.SubStorage{ { Storage: smallFileStorage, - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index bf00c4289..d5bf8429c 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestHeadRaw(t *testing.T) { defer os.RemoveAll(t.Name()) cnr := cidtest.ID() - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() parent := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(parent, "foo", "bar") @@ -70,7 +70,7 @@ func TestHeadRaw(t *testing.T) { _, err = e.Head(context.Background(), headPrm) require.Error(t, err) - var si *object.SplitInfoError + var si *objectSDK.SplitInfoError require.ErrorAs(t, err, &si) // SplitInfoError should contain info from both shards diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 7796913ea..a7f0a1552 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" @@ -83,7 +83,7 @@ func TestLockUserScenario(t *testing.T) { lockerAddr.SetContainer(cnr) lockerAddr.SetObject(lockerID) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) @@ -105,9 +105,9 @@ func TestLockUserScenario(t *testing.T) { require.NoError(t, err) // 2. - var locker object.Lock + var locker objectSDK.Lock locker.WriteMembers([]oid.ID{id}) - object.WriteLock(lockerObj, locker) + objectSDK.WriteLock(lockerObj, locker) err = Put(context.Background(), e, lockerObj) require.NoError(t, err) @@ -123,7 +123,7 @@ func TestLockUserScenario(t *testing.T) { require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 4. - tombObj.SetType(object.TypeTombstone) + tombObj.SetType(objectSDK.TypeTombstone) tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) @@ -188,12 +188,12 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, err) // 2. - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lock.SetAttributes(a) err = Put(context.Background(), e, lock) @@ -266,7 +266,7 @@ func TestLockForceRemoval(t *testing.T) { // 2. lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) err = Put(context.Background(), e, lock) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 6a7bf1b7f..6a8c9fab9 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -15,7 +15,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -29,7 +29,7 @@ func (p *SelectPrm) WithContainerID(cnr cid.ID) { } // WithFilters is a Select option to set the object filters. -func (p *SelectPrm) WithFilters(fs object.SearchFilters) { +func (p *SelectPrm) WithFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -144,7 +144,7 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro } // Select selects objects from local storage using provided filters. -func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm selectPrm.WithContainerID(cnr) selectPrm.WithFilters(fs) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 4cd35a088..f1650b5ae 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func BenchmarkTreeVsSearch(b *testing.B) { @@ -47,8 +47,8 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { var prm SelectPrm prm.WithContainerID(cid) - var fs object.SearchFilters - fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), object.MatchStringEqual) + var fs objectSDK.SearchFilters + fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) prm.WithFilters(fs) for i := 0; i < b.N; i++ { diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 059f00b3a..383c596af 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -6,7 +6,7 @@ import ( "testing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" @@ -48,7 +48,7 @@ func (g RandAddrGenerator) Next() oid.Address { // ObjectGenerator is the interface of types that generate object entries. type ObjectGenerator interface { - Next() *object.Object + Next() *objectSDK.Object } // SeqObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and sequential IDs. @@ -59,7 +59,7 @@ type SeqObjGenerator struct { var _ ObjectGenerator = &SeqObjGenerator{} -func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *object.Object { +func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *objectSDK.Object { data := make([]byte, sz) _, _ = rand.Read(data) obj := GenerateObjectWithCIDWithPayload(cid, data) @@ -67,7 +67,7 @@ func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *ob return obj } -func (g *SeqObjGenerator) Next() *object.Object { +func (g *SeqObjGenerator) Next() *objectSDK.Object { var id oid.ID binary.LittleEndian.PutUint64(id[:], g.cnt.Add(1)) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) @@ -80,7 +80,7 @@ type RandObjGenerator struct { var _ ObjectGenerator = &RandObjGenerator{} -func (g *RandObjGenerator) Next() *object.Object { +func (g *RandObjGenerator) Next() *objectSDK.Object { var id oid.ID _, _ = rand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) @@ -92,13 +92,13 @@ type OverwriteObjGenerator struct { MaxObjects uint64 } -func (g *OverwriteObjGenerator) Next() *object.Object { +func (g *OverwriteObjGenerator) Next() *objectSDK.Object { var id oid.ID binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.MaxObjects)))) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } -func AddressFromObject(t testing.TB, obj *object.Object) oid.Address { +func AddressFromObject(t testing.TB, obj *objectSDK.Object) oid.Address { var addr oid.Address id, isSet := obj.ID() diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 4f6d95816..9cbce27bf 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" @@ -16,17 +16,17 @@ import ( const defaultDataSize = 32 -func GenerateObject() *object.Object { +func GenerateObject() *objectSDK.Object { return GenerateObjectWithCID(cidtest.ID()) } -func GenerateObjectWithCID(cnr cid.ID) *object.Object { +func GenerateObjectWithCID(cnr cid.ID) *objectSDK.Object { data := make([]byte, defaultDataSize) _, _ = rand.Read(data) return GenerateObjectWithCIDWithPayload(cnr, data) } -func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { +func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *objectSDK.Object { var ver version.Version ver.SetMajor(2) ver.SetMinor(1) @@ -37,7 +37,7 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { var csumTZ checksum.Checksum csumTZ.SetTillichZemor(tz.Sum(csum.Value())) - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cnr) @@ -49,8 +49,8 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { return obj } -func AddAttribute(obj *object.Object, key, val string) { - var attr object.Attribute +func AddAttribute(obj *objectSDK.Object, key, val string) { + var attr objectSDK.Attribute attr.SetKey(key) attr.SetValue(val) @@ -59,7 +59,7 @@ func AddAttribute(obj *object.Object, key, val string) { obj.SetAttributes(attrs...) } -func AddPayload(obj *object.Object, size int) { +func AddPayload(obj *objectSDK.Object, size int) { buf := make([]byte, size) _, _ = rand.Read(buf) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 507bfcd89..2c5fa0233 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -8,7 +8,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -34,7 +34,7 @@ func TestCounters(t *testing.T) { t.Run("put", func(t *testing.T) { t.Parallel() db := newDB(t) - oo := make([]*object.Object, 0, objCount) + oo := make([]*objectSDK.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) } @@ -191,7 +191,7 @@ func TestCounters_Expired(t *testing.T) { oo := make([]oid.Address, objCount) for i := range oo { - oo[i] = putWithExpiration(t, db, object.TypeRegular, epoch+1) + oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1) } // 1. objects are available and counters are correct @@ -275,12 +275,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(len(oo)), c.Logic()) } -func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Object { +func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object { var prm meta.PutPrm var err error parent := testutil.GenerateObject() - oo := make([]*object.Object, 0, count) + oo := make([]*objectSDK.Object, 0, count) for i := 0; i < count; i++ { o := testutil.GenerateObject() if withParent { diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index e7de46e15..f5341ff2e 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -14,7 +14,7 @@ import ( v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/mr-tron/base58" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -39,7 +39,7 @@ type DB struct { modeMtx sync.RWMutex mode mode.Mode - matchers map[object.SearchMatchType]matcher + matchers map[objectSDK.SearchMatchType]matcher boltDB *bbolt.DB @@ -89,20 +89,20 @@ func New(opts ...Option) *DB { return &DB{ cfg: c, - matchers: map[object.SearchMatchType]matcher{ - object.MatchUnknown: { + matchers: map[objectSDK.SearchMatchType]matcher{ + objectSDK.MatchUnknown: { matchSlow: unknownMatcher, matchBucket: unknownMatcherBucket, }, - object.MatchStringEqual: { + objectSDK.MatchStringEqual: { matchSlow: stringEqualMatcher, matchBucket: stringEqualMatcherBucket, }, - object.MatchStringNotEqual: { + objectSDK.MatchStringNotEqual: { matchSlow: stringNotEqualMatcher, matchBucket: stringNotEqualMatcherBucket, }, - object.MatchCommonPrefix: { + objectSDK.MatchCommonPrefix: { matchSlow: stringCommonPrefixMatcher, matchBucket: stringCommonPrefixMatcherBucket, }, @@ -276,7 +276,7 @@ func bucketKeyHelper(hdr string, val string) []byte { return v case v2object.FilterHeaderSplitID: - s := object.NewSplitID() + s := objectSDK.NewSplitID() err := s.Parse(val) if err != nil { diff --git a/pkg/local_object_storage/metabase/expired_test.go b/pkg/local_object_storage/metabase/expired_test.go index 5755b5cdb..9a6bcc5db 100644 --- a/pkg/local_object_storage/metabase/expired_test.go +++ b/pkg/local_object_storage/metabase/expired_test.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -77,7 +77,7 @@ func TestDB_SelectExpired(t *testing.T) { require.Contains(t, expired, getAddressSafe(t, expiredObj22)) } -func getAddressSafe(t *testing.T, o *object.Object) oid.Address { +func getAddressSafe(t *testing.T, o *objectSDK.Object) oid.Address { cid, set := o.ContainerID() if !set { t.Fatalf("container id required") diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 76744aa33..7ba5a68a2 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -276,7 +276,7 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool return false, garbageBKT.Put(key, zeroValue) } -func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *object.Object, res *InhumeRes) error { +func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { res.availableImhumed++ @@ -285,7 +285,7 @@ func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Buc // if object is stored, and it is regular object then update bucket // with container size estimations - if obj.Type() == object.TypeRegular { + if obj.Type() == objectSDK.TypeRegular { err := changeContainerSize(tx, cnr, obj.PayloadSize(), false) if err != nil { return err diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 78bfd2914..a1e21ef25 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -12,7 +12,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -21,13 +22,13 @@ import ( // ExpiredObject is a descriptor of expired object from DB. type ExpiredObject struct { - typ object.Type + typ objectSDK.Type addr oid.Address } // Type returns type of the expired object. -func (e *ExpiredObject) Type() object.Type { +func (e *ExpiredObject) Type() objectSDK.Type { return e.typ } diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index e07184eb6..034a931d2 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -9,7 +9,7 @@ import ( object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -20,19 +20,19 @@ func TestDB_IterateExpired(t *testing.T) { const epoch = 13 - mAlive := map[object.Type]oid.Address{} - mExpired := map[object.Type]oid.Address{} + mAlive := map[objectSDK.Type]oid.Address{} + mExpired := map[objectSDK.Type]oid.Address{} - for _, typ := range []object.Type{ - object.TypeRegular, - object.TypeTombstone, - object.TypeLock, + for _, typ := range []objectSDK.Type{ + objectSDK.TypeRegular, + objectSDK.TypeTombstone, + objectSDK.TypeLock, } { mAlive[typ] = putWithExpiration(t, db, typ, epoch) mExpired[typ] = putWithExpiration(t, db, typ, epoch-1) } - expiredLocked := putWithExpiration(t, db, object.TypeRegular, epoch-1) + expiredLocked := putWithExpiration(t, db, objectSDK.TypeRegular, epoch-1) require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) @@ -56,7 +56,7 @@ func TestDB_IterateExpired(t *testing.T) { require.Empty(t, mExpired) } -func putWithExpiration(t *testing.T, db *meta.DB, typ object.Type, expiresAt uint64) oid.Address { +func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt uint64) oid.Address { obj := testutil.GenerateObject() obj.SetType(typ) testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(expiresAt, 10)) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 337265318..37a574a02 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -124,15 +124,15 @@ loop: continue } - var objType object.Type + var objType objectSDK.Type switch prefix { case primaryPrefix: - objType = object.TypeRegular + objType = objectSDK.TypeRegular case lockersPrefix: - objType = object.TypeLock + objType = objectSDK.TypeLock case tombstonePrefix: - objType = object.TypeTombstone + objType = objectSDK.TypeTombstone default: continue } @@ -175,7 +175,7 @@ loop: // selectNFromBucket similar to selectAllFromBucket but uses cursor to find // object to start selecting from. Ignores inhumed objects. func selectNFromBucket(bkt *bbolt.Bucket, // main bucket - objType object.Type, // type of the objects stored in the main bucket + objType objectSDK.Type, // type of the objects stored in the main bucket graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets cidRaw []byte, // container ID prefix, optimization cnt cid.ID, // container ID diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index f3388daf0..50aac223b 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -79,7 +79,7 @@ func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { key := make([]byte, cidSize) return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { - if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != object.TypeRegular { + if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != objectSDK.TypeRegular { return logicerr.Wrap(apistatus.LockNonRegularObject{}) } diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index d5e063431..b442296fd 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -9,7 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" @@ -28,10 +28,10 @@ func TestDB_Lock(t *testing.T) { }) t.Run("(ir)regular", func(t *testing.T) { - for _, typ := range [...]object.Type{ - object.TypeTombstone, - object.TypeLock, - object.TypeRegular, + for _, typ := range [...]objectSDK.Type{ + objectSDK.TypeTombstone, + objectSDK.TypeLock, + objectSDK.TypeRegular, } { obj := objecttest.Object() obj.SetType(typ) @@ -47,7 +47,7 @@ func TestDB_Lock(t *testing.T) { // try to lock it err = db.Lock(context.Background(), cnr, oidtest.ID(), []oid.ID{id}) - if typ == object.TypeRegular { + if typ == objectSDK.TypeRegular { require.NoError(t, err, typ) } else { require.ErrorAs(t, err, &e, typ) @@ -180,7 +180,7 @@ func TestDB_Lock_Expired(t *testing.T) { db := newDB(t, meta.WithEpochState(es)) // put an object - addr := putWithExpiration(t, db, object.TypeRegular, 124) + addr := putWithExpiration(t, db, objectSDK.TypeRegular, 124) // expire the obj es.e = 125 @@ -242,10 +242,10 @@ func TestDB_IsLocked(t *testing.T) { } // putAndLockObj puts object, returns it and its locker. -func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Object, *object.Object) { +func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*objectSDK.Object, *objectSDK.Object) { cnr := cidtest.ID() - lockedObjs := make([]*object.Object, 0, numOfLockedObjs) + lockedObjs := make([]*objectSDK.Object, 0, numOfLockedObjs) lockedObjIDs := make([]oid.ID, 0, numOfLockedObjs) for i := 0; i < numOfLockedObjs; i++ { @@ -261,7 +261,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob lockObj := testutil.GenerateObjectWithCID(cnr) lockID, _ := lockObj.ID() - lockObj.SetType(object.TypeLock) + lockObj.SetType(objectSDK.TypeLock) err := putBig(db, lockObj) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 998ce9b94..6a7acd7cc 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -31,14 +31,14 @@ type ( cnr cid.ID - fastFilters, slowFilters object.SearchFilters + fastFilters, slowFilters objectSDK.SearchFilters } ) // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -52,7 +52,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -98,7 +98,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err })) } -func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, currEpoch uint64) ([]oid.Address, error) { +func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters, currEpoch uint64) ([]oid.Address, error) { group, err := groupFilters(fs) if err != nil { return nil, err @@ -186,7 +186,7 @@ func selectAllFromBucket(tx *bbolt.Tx, name []byte, to map[string]int, fNum int) func (db *DB) selectFastFilter( tx *bbolt.Tx, cnr cid.ID, // container we search on - f object.SearchFilter, // fast filter + f objectSDK.SearchFilter, // fast filter to map[string]int, // resulting cache fNum int, // index of filter ) { @@ -220,7 +220,7 @@ func (db *DB) selectFastFilter( default: // user attribute bucketName := attributeBucketName(cnr, f.Header(), bucketName) - if f.Operation() == object.MatchNotPresent { + if f.Operation() == objectSDK.MatchNotPresent { selectOutsideFKBT(tx, allBucketNames(cnr), bucketName, to, fNum) } else { db.selectFromFKBT(tx, bucketName, f, to, fNum) @@ -244,7 +244,7 @@ func allBucketNames(cnr cid.ID) (names [][]byte) { return } -func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string) (names [][]byte) { +func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal string) (names [][]byte) { appendNames := func(key string) { fns, ok := mBucketNaming[key] if ok { @@ -256,15 +256,15 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string switch mType { default: - case object.MatchStringNotEqual: + case objectSDK.MatchStringNotEqual: for key := range mBucketNaming { if key != typeVal { appendNames(key) } } - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: appendNames(typeVal) - case object.MatchCommonPrefix: + case objectSDK.MatchCommonPrefix: for key := range mBucketNaming { if strings.HasPrefix(key, typeVal) { appendNames(key) @@ -280,7 +280,7 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string func (db *DB) selectFromFKBT( tx *bbolt.Tx, name []byte, // fkbt root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -357,7 +357,7 @@ func selectOutsideFKBT( func (db *DB) selectFromList( tx *bbolt.Tx, name []byte, // list root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -372,7 +372,7 @@ func (db *DB) selectFromList( ) switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value()))) if err != nil { db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error())) @@ -416,7 +416,7 @@ func (db *DB) selectFromList( // selectObjectID processes objectID filter with in-place optimizations. func (db *DB) selectObjectID( tx *bbolt.Tx, - f object.SearchFilter, + f objectSDK.SearchFilter, cnr cid.ID, to map[string]int, // resulting cache fNum int, // index of filter @@ -436,7 +436,7 @@ func (db *DB) selectObjectID( } switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: var id oid.ID if err := id.DecodeString(f.Value()); err == nil { appendOID(id) @@ -451,7 +451,7 @@ func (db *DB) selectObjectID( return } - for _, bucketName := range bucketNamesForType(cnr, object.MatchStringNotEqual, "") { + for _, bucketName := range bucketNamesForType(cnr, objectSDK.MatchStringNotEqual, "") { // copy-paste from DB.selectAllFrom bkt := tx.Bucket(bucketName) if bkt == nil { @@ -475,7 +475,7 @@ func (db *DB) selectObjectID( } // matchSlowFilters return true if object header is matched by all slow filters. -func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFilters, currEpoch uint64) bool { +func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) bool { if len(f) == 0 { return true } @@ -521,10 +521,10 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFi // groupFilters divides filters in two groups: fast and slow. Fast filters // processed by indexes and slow filters processed after by unmarshaling // object headers. -func groupFilters(filters object.SearchFilters) (filterGroup, error) { +func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { res := filterGroup{ - fastFilters: make(object.SearchFilters, 0, len(filters)), - slowFilters: make(object.SearchFilters, 0, len(filters)), + fastFilters: make(objectSDK.SearchFilters, 0, len(filters)), + slowFilters: make(objectSDK.SearchFilters, 0, len(filters)), } for i := range filters { @@ -557,9 +557,9 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) { } // Returns true if at least 1 object can satisfy fs. -func checkNonEmpty(fs object.SearchFilters) bool { +func checkNonEmpty(fs objectSDK.SearchFilters) bool { for i := range fs { - if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) { + if fs[i].Operation() == objectSDK.MatchNotPresent && isSystemKey(fs[i].Header()) { return true } } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index c50fd051f..4e58ec20b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -6,7 +6,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -120,7 +120,7 @@ const ( addressKeySize = cidSize + objectKeySize ) -var splitInfoError *object.SplitInfoError // for errors.As comparisons +var splitInfoError *objectSDK.SplitInfoError // for errors.As comparisons func bucketName(cnr cid.ID, prefix byte, key []byte) []byte { key[0] = prefix @@ -221,7 +221,7 @@ func objectKey(obj oid.ID, key []byte) []byte { // if meets irregular object container in objs - returns its type, otherwise returns object.TypeRegular. // // firstIrregularObjectType(tx, cnr, obj) usage allows getting object type. -func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object.Type { +func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) objectSDK.Type { if len(objs) == 0 { panic("empty object list in firstIrregularObjectType") } @@ -229,11 +229,11 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object var keys [2][1 + cidSize]byte irregularTypeBuckets := [...]struct { - typ object.Type + typ objectSDK.Type name []byte }{ - {object.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, - {object.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, + {objectSDK.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, + {objectSDK.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, } for i := range objs { @@ -244,7 +244,7 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object } } - return object.TypeRegular + return objectSDK.TypeRegular } // return true if provided object is of LOCK type. diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index b8d516871..2221d57c1 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -335,7 +335,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() != object.TypeTombstone && o.Type() != object.TypeLock { + if o.Type() != objectSDK.TypeTombstone && o.Type() != objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { @@ -519,7 +519,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { batch := make([]oid.Address, 0, batchSize) expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() == object.TypeLock { + if o.Type() == objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index d94029976..bc895d67b 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" @@ -41,7 +41,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index dd21745cc..bc13c622b 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -9,7 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -87,7 +87,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { return res, fmt.Errorf("can't list stored containers: %w", err) } - filters := object.NewSearchFilters() + filters := objectSDK.NewSearchFilters() filters.AddPhyFilter() for i := range lst { diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 75010179a..61f2bb9f0 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -38,7 +38,7 @@ func TestShard_Lock(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(2), blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, @@ -70,7 +70,7 @@ func TestShard_Lock(t *testing.T) { objID, _ := obj.ID() lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lockID, _ := lock.ID() // put the object diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index d76c240b7..ad0fc15ca 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -96,7 +96,7 @@ func TestCounters(t *testing.T) { require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 - oo := make([]*object.Object, objNumber) + oo := make([]*objectSDK.Object, objNumber) for i := 0; i < objNumber; i++ { oo[i] = testutil.GenerateObject() } @@ -240,7 +240,7 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { return sh, mm } -func addrFromObjs(oo []*object.Object) []oid.Address { +func addrFromObjs(oo []*objectSDK.Object) []oid.Address { aa := make([]oid.Address, len(oo)) for i := 0; i < len(oo); i++ { diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 79dc4846e..688b7aae7 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -17,14 +17,14 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - obj *object.Object + obj *objectSDK.Object } // PutRes groups the resulting values of Put operation. type PutRes struct{} // SetObject is a Put option to set object to save. -func (p *PutPrm) SetObject(obj *object.Object) { +func (p *PutPrm) SetObject(obj *objectSDK.Object) { p.obj = obj } diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 362fd86c5..e94482e36 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -29,7 +29,7 @@ type RngPrm struct { // RngRes groups the resulting values of GetRange operation. type RngRes struct { - obj *object.Object + obj *objectSDK.Object hasMeta bool } @@ -54,7 +54,7 @@ func (p *RngPrm) SetIgnoreMeta(ignore bool) { // Object returns the requested object part. // // Instance payload contains the requested range of the original object. -func (r RngRes) Object() *object.Object { +func (r RngRes) Object() *objectSDK.Object { return r.obj } @@ -71,7 +71,7 @@ func (r RngRes) HasMeta() bool { // Returns ErrRangeOutOfBounds if the requested object range is out of bounds. // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. -// Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the objectSDK.ErrObjectIsExpired if the object is presented but already expired. // Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", @@ -91,7 +91,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return RngRes{}, ErrShardDisabled } - cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr getRngPrm.Range.SetOffset(prm.off) @@ -103,13 +103,13 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, err } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(res.Data) return obj, nil } - wc := func(c writecache.Cache) (*object.Object, error) { + wc := func(c writecache.Cache) (*objectSDK.Object, error) { res, err := c.Get(ctx, prm.addr) if err != nil { return nil, err @@ -122,7 +122,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(payload[from:to]) return obj, nil } diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 2d4d473b4..1615f5fbe 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -7,7 +7,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -16,7 +16,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -30,7 +30,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 7b2fdb5d1..a9a8e4ea7 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" @@ -56,7 +56,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, diff --git a/pkg/local_object_storage/util/splitinfo.go b/pkg/local_object_storage/util/splitinfo.go index 4a6d22268..6ae1c3e46 100644 --- a/pkg/local_object_storage/util/splitinfo.go +++ b/pkg/local_object_storage/util/splitinfo.go @@ -1,12 +1,12 @@ package util import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // MergeSplitInfo ignores conflicts and rewrites `to` with non empty values // from `from`. -func MergeSplitInfo(from, to *object.SplitInfo) *object.SplitInfo { +func MergeSplitInfo(from, to *objectSDK.SplitInfo) *objectSDK.SplitInfo { to.SetSplitID(from.SplitID()) // overwrite SplitID and ignore conflicts if lp, ok := from.LastPart(); ok { diff --git a/pkg/local_object_storage/util/splitinfo_test.go b/pkg/local_object_storage/util/splitinfo_test.go index a0626db28..642fef3b8 100644 --- a/pkg/local_object_storage/util/splitinfo_test.go +++ b/pkg/local_object_storage/util/splitinfo_test.go @@ -5,7 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -15,7 +15,7 @@ func TestMergeSplitInfo(t *testing.T) { uid, err := uuid.NewUUID() require.NoError(t, err) - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() splitID.SetUUID(uid) var rawLinkID, rawLastID [32]byte @@ -30,35 +30,35 @@ func TestMergeSplitInfo(t *testing.T) { require.NoError(t, err) lastID.SetSHA256(rawLastID) - target := object.NewSplitInfo() // target is SplitInfo struct with all fields set + target := objectSDK.NewSplitInfo() // target is SplitInfo struct with all fields set target.SetSplitID(splitID) target.SetLastPart(lastID) target.SetLink(linkID) t.Run("merge empty", func(t *testing.T) { - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() result := util.MergeSplitInfo(target, to) require.Equal(t, result, target) }) t.Run("merge link", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLastPart(lastID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLink(linkID) result := util.MergeSplitInfo(from, to) require.Equal(t, result, target) }) t.Run("merge last", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLink(linkID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLastPart(lastID) result := util.MergeSplitInfo(from, to) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 21f968b04..779bac39a 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -129,7 +129,7 @@ func (c *cache) flushSmallObjects() { var count int for i := range m { - obj := object.New() + obj := objectSDK.New() if err := obj.Unmarshal(m[i].data); err != nil { continue } @@ -201,7 +201,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - var obj object.Object + var obj objectSDK.Object err = obj.Unmarshal(data) if err != nil { c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) @@ -231,7 +231,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { func (c *cache) workerFlushSmall() { defer c.wg.Done() - var obj *object.Object + var obj *objectSDK.Object for { // Give priority to direct put. select { @@ -251,7 +251,7 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st StorageType) error { +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { var err error defer func() { @@ -322,7 +322,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - var obj object.Object + var obj objectSDK.Object if err := obj.Unmarshal(data); err != nil { c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index e8224ce5d..2223bef02 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" @@ -28,7 +28,7 @@ import ( type objectPair struct { addr oid.Address - obj *object.Object + obj *objectSDK.Object } func TestFlush(t *testing.T) { @@ -211,14 +211,14 @@ func putObject(t *testing.T, c Cache, size int) objectPair { } -func newObject(t *testing.T, size int) (*object.Object, []byte) { - obj := object.New() +func newObject(t *testing.T, size int) (*objectSDK.Object, []byte) { + obj := objectSDK.New() ver := versionSDK.Current() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cidtest.ID()) - obj.SetType(object.TypeRegular) + obj.SetType(objectSDK.TypeRegular) obj.SetVersion(&ver) obj.SetPayloadChecksum(checksumtest.Checksum()) obj.SetPayloadHomomorphicHash(checksumtest.Checksum()) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 962c9c39a..067ff5ae5 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -24,8 +24,8 @@ type Info struct { // Cache represents write-cache for objects. type Cache interface { - Get(ctx context.Context, address oid.Address) (*object.Object, error) - Head(context.Context, oid.Address) (*object.Object, error) + Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) // Delete removes object referenced by the given oid.Address from the // Cache. Returns any error encountered that prevented the object to be // removed. @@ -58,7 +58,7 @@ type cache struct { compressFlags map[string]struct{} // flushCh is a channel with objects to flush. - flushCh chan *object.Object + flushCh chan *objectSDK.Object // closeCh is close channel, protected by modeMtx. closeCh chan struct{} // wg is a wait group for flush workers. @@ -75,7 +75,7 @@ const wcStorageType = "write-cache" type objectInfo struct { addr string data []byte - obj *object.Object + obj *objectSDK.Object } const ( @@ -91,7 +91,7 @@ var ( // New creates new writecache instance. func New(opts ...Option) Cache { c := &cache{ - flushCh: make(chan *object.Object), + flushCh: make(chan *objectSDK.Object), mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b7f7dc20f..18155849b 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -13,7 +13,7 @@ import ( metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -166,7 +166,7 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) if success || firstErr == nil || errors.Is(firstErr, errRecentlyFailed) { @@ -195,7 +195,7 @@ func (x *multiClient) ReportError(err error) { // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError if errors.As(err, &siErr) { return } diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index ce5d98d5d..6f879123b 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -22,12 +22,12 @@ type testLocalStorage struct { expAddr oid.Address - obj *object.Object + obj *objectSDK.Object err error } -func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*object.Object, error) { +func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { require.True(s.t, addr.Container().Equals(s.expAddr.Container())) require.True(s.t, addr.Object().Equals(s.expAddr.Object())) @@ -69,11 +69,11 @@ func TestHeadRequest(t *testing.T) { meta.SetXHeaders(xHdrs) - obj := object.New() + obj := objectSDK.New() attrKey := "attr_key" attrVal := "attr_val" - var attr object.Attribute + var attr objectSDK.Attribute attr.SetKey(attrKey) attr.SetValue(attrVal) obj.SetAttributes(attr) diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 095810848..c7a1b7729 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -28,7 +28,7 @@ type cfg struct { } type ObjectStorage interface { - Head(context.Context, oid.Address) (*object.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) } type Request interface { @@ -141,7 +141,7 @@ func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.SearchRequest: cnrV2 := req.GetBody().GetContainerID() @@ -171,7 +171,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.HeadResponse: oV2 := new(objectV2.Object) @@ -197,7 +197,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetHeader(hdr) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } return nil } diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go index 0a63981cb..690e4aa70 100644 --- a/pkg/services/object/acl/eacl/v2/object.go +++ b/pkg/services/object/acl/eacl/v2/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -26,7 +26,7 @@ func u64Value(v uint64) string { return strconv.FormatUint(v, 10) } -func headersFromObject(obj *object.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { +func headersFromObject(obj *objectSDK.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { var count int for obj := obj; obj != nil; obj = obj.Parent() { count += 9 + len(obj.Attributes()) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 343efbc04..b10f045ee 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -29,11 +29,11 @@ type execCtx struct { log *logger.Logger - tombstone *object.Tombstone + tombstone *objectSDK.Tombstone - splitInfo *object.SplitInfo + splitInfo *objectSDK.SplitInfo - tombstoneObj *object.Object + tombstoneObj *objectSDK.Object } const ( @@ -241,9 +241,9 @@ func (exec *execCtx) initTombstoneObject() bool { return false } - exec.tombstoneObj = object.New() + exec.tombstoneObj = objectSDK.New() exec.tombstoneObj.SetContainerID(exec.containerID()) - exec.tombstoneObj.SetType(object.TypeTombstone) + exec.tombstoneObj.SetType(objectSDK.TypeTombstone) exec.tombstoneObj.SetPayload(payload) tokenSession := exec.commonParameters().SessionToken() @@ -256,7 +256,7 @@ func (exec *execCtx) initTombstoneObject() bool { exec.tombstoneObj.SetOwnerID(&localUser) } - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.FormatUint(exec.tombstone.ExpirationEpoch(), 10)) diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 34839b194..ad3e10bc6 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -4,7 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -35,7 +35,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { return false } - exec.tombstone = object.NewTombstone() + exec.tombstone = objectSDK.NewTombstone() exec.tombstone.SetExpirationEpoch( exec.svc.netInfo.CurrentEpoch() + tsLifetime, ) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 11ff13b45..5c09ad123 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -9,7 +9,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" @@ -41,7 +41,7 @@ type cfg struct { header interface { // must return (nil, nil) for PHY objects - splitInfo(context.Context, *execCtx) (*object.SplitInfo, error) + splitInfo(context.Context, *execCtx) (*objectSDK.SplitInfo, error) children(context.Context, *execCtx) ([]oid.ID, error) @@ -85,7 +85,7 @@ func New(opts ...Option) *Service { // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Delete service"))} + c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} } } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index f6341f02a..b8e8e6324 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -7,7 +7,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -21,7 +21,7 @@ type simpleIDWriter struct { ids []oid.ID } -func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*object.Object, error) { +func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*objectSDK.Object, error) { wr := getsvc.NewSimpleObjectWriter() p := getsvc.HeadPrm{} @@ -38,10 +38,10 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi return wr.Object(), nil } -func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*object.SplitInfo, error) { +func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*objectSDK.SplitInfo, error) { _, err := w.headAddress(ctx, exec, exec.address()) - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError switch { case err == nil: @@ -83,8 +83,8 @@ func (w *headSvcWrapper) previous(ctx context.Context, exec *execCtx, id oid.ID) } func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]oid.ID, error) { - fs := object.SearchFilters{} - fs.AddSplitIDFilter(object.MatchStringEqual, exec.splitInfo.SplitID()) + fs := objectSDK.SearchFilters{} + fs.AddSplitIDFilter(objectSDK.MatchStringEqual, exec.splitInfo.SplitID()) wr := new(simpleIDWriter) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 457193a59..e3037a70b 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) @@ -74,7 +74,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { localStorage: s.localStorage, prm: prm, - infoSplit: object.NewSplitInfo(), + infoSplit: objectSDK.NewSplitInfo(), } exec.setLogger(s.log) diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 580c0b58c..40aa3f62e 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -31,7 +31,7 @@ type getRequestForwarder struct { Stream *streamObjectWriter } -func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -85,7 +85,7 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb var err error f.OnceHeaderSending.Do(func() { - err = f.Stream.WriteHeader(ctx, object.NewFromV2(obj)) + err = f.Stream.WriteHeader(ctx, objectSDK.NewFromV2(obj)) }) if err != nil { return errCouldNotWriteObjHeader(err) @@ -164,8 +164,8 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 7858e2b53..8a56c59a6 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -30,7 +30,7 @@ type getRangeRequestForwarder struct { Stream *streamObjectRangeWriter } -func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRangeRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -130,8 +130,8 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index fa1506435..a1bce1517 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -30,7 +30,7 @@ type headRequestForwarder struct { Key *ecdsa.PrivateKey } -func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "headRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -82,15 +82,15 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne return nil, err } case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return nil, object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return nil, objectSDK.NewSplitInfoError(si) } objv2 := new(objectV2.Object) objv2.SetHeader(hdr) objv2.SetSignature(idSig) - obj := object.NewFromV2(objv2) + obj := objectSDK.NewFromV2(objv2) obj.SetID(f.ObjectAddr.Object()) return obj, nil diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 1bd8befaf..d4bce178a 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -8,7 +8,7 @@ import ( objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // Service implements Get operation of Object service v2. @@ -47,7 +47,7 @@ func (s *Service) Get(req *objectV2.GetRequest, stream objectSvc.GetObjectStream err = s.svc.Get(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -66,7 +66,7 @@ func (s *Service) GetRange(req *objectV2.GetRangeRequest, stream objectSvc.GetOb err = s.svc.GetRange(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -103,7 +103,7 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV err = s.svc.Head(ctx, *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError if errors.As(err, &splitErr) { setSplitInfoHeadResponse(splitErr.SplitInfo(), resp) diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index 4347ef416..ce9a5c767 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -5,7 +5,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type streamObjectWriter struct { @@ -16,7 +16,7 @@ type streamObjectRangeWriter struct { objectSvc.GetObjectRangeStream } -func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { p := new(objectV2.GetObjectPartInit) objV2 := obj.ToV2() diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 519d9afa0..9ecc9167f 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -16,7 +16,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -98,7 +98,7 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get p.WithAddress(addr) p.WithRawFlag(body.GetRaw()) p.SetChunkWriter(streamWrapper) - p.SetRange(object.NewRangeFromV2(body.GetRange())) + p.SetRange(objectSDK.NewRangeFromV2(body.GetRange())) err = p.Validate() if err != nil { @@ -167,10 +167,10 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran } rngsV2 := body.GetRanges() - rngs := make([]object.Range, len(rngsV2)) + rngs := make([]objectSDK.Range, len(rngsV2)) for i := range rngsV2 { - rngs[i] = *object.NewRangeFromV2(&rngsV2[i]) + rngs[i] = *objectSDK.NewRangeFromV2(&rngsV2[i]) } p.SetRangeList(rngs) @@ -198,7 +198,7 @@ type headResponseWriter struct { body *objectV2.HeadResponseBody } -func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *object.Object) error { +func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *objectSDK.Object) error { if w.mainOnly { w.body.SetHeaderPart(toShortObjectHeader(hdr)) } else { @@ -259,7 +259,7 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon return p, nil } -func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { +func splitInfoResponse(info *objectSDK.SplitInfo) *objectV2.GetResponse { resp := new(objectV2.GetResponse) body := new(objectV2.GetResponseBody) @@ -270,7 +270,7 @@ func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { return resp } -func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { +func splitInfoRangeResponse(info *objectSDK.SplitInfo) *objectV2.GetRangeResponse { resp := new(objectV2.GetRangeResponse) body := new(objectV2.GetRangeResponseBody) @@ -281,7 +281,7 @@ func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { return resp } -func setSplitInfoHeadResponse(info *object.SplitInfo, resp *objectV2.HeadResponse) { +func setSplitInfoHeadResponse(info *objectSDK.SplitInfo, resp *objectV2.HeadResponse) { resp.GetBody().SetHeaderPart(info.ToV2()) } @@ -297,7 +297,7 @@ func toHashResponse(typ refs.ChecksumType, res *getsvc.RangeHashRes) *objectV2.G return resp } -func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toFullObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { obj := hdr.ToV2() hs := new(objectV2.HeaderWithSignature) @@ -307,7 +307,7 @@ func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return hs } -func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toShortObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { hdrV2 := hdr.ToV2().GetHeader() sh := new(objectV2.ShortHeader) @@ -322,11 +322,11 @@ func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return sh } -func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*object.Object, error)) getsvc.RequestForwarder { - return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*object.Object, error) { +func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*objectSDK.Object, error)) getsvc.RequestForwarder { + return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*objectSDK.Object, error) { var ( firstErr error - res *object.Object + res *objectSDK.Object key = info.PublicKey() ) diff --git a/pkg/services/object/get/writer.go b/pkg/services/object/get/writer.go index 78af5db41..1b842adeb 100644 --- a/pkg/services/object/get/writer.go +++ b/pkg/services/object/get/writer.go @@ -4,7 +4,7 @@ import ( "context" "io" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // ChunkWriter is an interface of target component @@ -16,7 +16,7 @@ type ChunkWriter interface { // HeaderWriter is an interface of target component // to write object header. type HeaderWriter interface { - WriteHeader(context.Context, *object.Object) error + WriteHeader(context.Context, *objectSDK.Object) error } // ObjectWriter is an interface of target component to write object. @@ -26,7 +26,7 @@ type ObjectWriter interface { } type SimpleObjectWriter struct { - obj *object.Object + obj *objectSDK.Object pld []byte } @@ -45,11 +45,11 @@ type hasherWrapper struct { func NewSimpleObjectWriter() *SimpleObjectWriter { return &SimpleObjectWriter{ - obj: object.New(), + obj: objectSDK.New(), } } -func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { s.obj = obj s.pld = make([]byte, 0, obj.PayloadSize()) @@ -62,7 +62,7 @@ func (s *SimpleObjectWriter) WriteChunk(_ context.Context, p []byte) error { return nil } -func (s *SimpleObjectWriter) Object() *object.Object { +func (s *SimpleObjectWriter) Object() *objectSDK.Object { if len(s.pld) > 0 { s.obj.SetPayload(s.pld) } @@ -74,7 +74,7 @@ func (w *partWriter) WriteChunk(ctx context.Context, p []byte) error { return w.chunkWriter.WriteChunk(ctx, p) } -func (w *partWriter) WriteHeader(ctx context.Context, o *object.Object) error { +func (w *partWriter) WriteHeader(ctx context.Context, o *objectSDK.Object) error { return w.headWriter.WriteHeader(ctx, o) } diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index bcba181f2..c9c17d4d8 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -10,7 +10,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -64,7 +64,7 @@ func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { } // Head requests object header from the remote node. -func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*object.Object, error) { +func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK.Object, error) { key, err := h.keyStorage.GetKey(nil) if err != nil { return nil, fmt.Errorf("(%T) could not receive private key: %w", h, err) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index a54f6cd01..cfab77efe 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -14,7 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -77,7 +77,7 @@ type readPrmCommon struct { commonPrm } -// SetNetmapEpoch sets the epoch number to be used to locate the object. +// SetNetmapEpoch sets the epoch number to be used to locate the objectSDK. // // By default current epoch on the server will be used. func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { @@ -111,11 +111,11 @@ func (x *GetObjectPrm) SetAddress(addr oid.Address) { // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - obj *object.Object + obj *objectSDK.Object } -// Object returns requested object. -func (x GetObjectRes) Object() *object.Object { +// Object returns requested objectSDK. +func (x GetObjectRes) Object() *objectSDK.Object { return x.obj } @@ -125,10 +125,10 @@ func (x GetObjectRes) Object() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // Returns: -// - error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// - error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // - error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// GetObject ignores the provided session if it is not related to the requested object. +// GetObject ignores the provided session if it is not related to the requested objectSDK. func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { // here we ignore session if it is opened for other object since such // request will almost definitely fail. The case can occur, for example, @@ -155,7 +155,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { return nil, fmt.Errorf("init object reading: %w", err) } - var obj object.Object + var obj objectSDK.Object if !rdr.ReadHeader(&obj) { res, err := rdr.Close() @@ -210,11 +210,11 @@ func (x *HeadObjectPrm) SetAddress(addr oid.Address) { // HeadObjectRes groups the resulting values of GetObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -225,10 +225,10 @@ func (x HeadObjectRes) Header() *object.Object { // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// HeadObject ignores the provided session if it is not related to the requested object. +// HeadObject ignores the provided session if it is not related to the requested objectSDK. func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -255,7 +255,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) return nil, fmt.Errorf("read object header from FrostFS: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !cliRes.ReadHeader(&hdr) { return nil, errors.New("missing object header in the response") @@ -296,7 +296,7 @@ func (x *PayloadRangePrm) SetAddress(addr oid.Address) { // SetRange range of the object payload to be read. // // Required parameter. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.cliPrm.SetOffset(rng.GetOffset()) x.ln = rng.GetLength() } @@ -323,11 +323,11 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed; // error of type *apistatus.ObjectOutOfRange if the requested range is too big. // -// PayloadRange ignores the provided session if it is not related to the requested object. +// PayloadRange ignores the provided session if it is not related to the requested objectSDK. func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -377,13 +377,13 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e type PutObjectPrm struct { commonPrm - obj *object.Object + obj *objectSDK.Object } // SetObject sets object to be stored. // // Required parameter. -func (x *PutObjectPrm) SetObject(obj *object.Object) { +func (x *PutObjectPrm) SetObject(obj *objectSDK.Object) { x.obj = obj } @@ -392,7 +392,7 @@ type PutObjectRes struct { id oid.ID } -// ID returns identifier of the stored object. +// ID returns identifier of the stored objectSDK. func (x PutObjectRes) ID() oid.ID { return x.id } @@ -464,7 +464,7 @@ func (x *SearchObjectsPrm) SetContainerID(id cid.ID) { } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(fs object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(fs objectSDK.SearchFilters) { x.cliPrm.SetFilters(fs) } diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index f07122729..be202892a 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -5,7 +5,7 @@ import ( "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) @@ -14,7 +14,7 @@ import ( type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(context.Context, *object.Object) error + Put(context.Context, *objectSDK.Object) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error @@ -28,11 +28,11 @@ type ObjectStorage interface { type localTarget struct { storage ObjectStorage - obj *object.Object + obj *objectSDK.Object meta objectCore.ContentMeta } -func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMeta) error { +func (t *localTarget) WriteObject(obj *objectSDK.Object, meta objectCore.ContentMeta) error { t.obj = obj t.meta = meta @@ -41,12 +41,12 @@ func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMet func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { switch t.meta.Type() { - case object.TypeTombstone: + case objectSDK.TypeTombstone: err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) } - case object.TypeLock: + case objectSDK.TypeLock: err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index c8d1b29a2..52a7c102c 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -7,13 +7,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type PutInitPrm struct { common *util.CommonPrm - hdr *object.Object + hdr *objectSDK.Object cnr containerSDK.Container @@ -34,7 +34,7 @@ func (p *PutInitPrm) WithCommonPrm(v *util.CommonPrm) *PutInitPrm { return p } -func (p *PutInitPrm) WithObject(v *object.Object) *PutInitPrm { +func (p *PutInitPrm) WithObject(v *objectSDK.Object) *PutInitPrm { if p != nil { p.hdr = v } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index bcc566b74..a5b3f643c 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -11,7 +11,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) @@ -22,7 +22,7 @@ type remoteTarget struct { nodeInfo clientcore.NodeInfo - obj *object.Object + obj *objectSDK.Object clientConstructor ClientConstructor } @@ -39,10 +39,10 @@ type RemoteSender struct { type RemotePutPrm struct { node netmap.NodeInfo - obj *object.Object + obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(obj *object.Object, _ objectcore.ContentMeta) error { +func (t *remoteTarget) WriteObject(obj *objectSDK.Object, _ objectcore.ContentMeta) error { t.obj = obj return nil @@ -89,7 +89,7 @@ func (p *RemotePutPrm) WithNodeInfo(v netmap.NodeInfo) *RemotePutPrm { } // WithObject sets transferred object. -func (p *RemotePutPrm) WithObject(v *object.Object) *RemotePutPrm { +func (p *RemotePutPrm) WithObject(v *objectSDK.Object) *RemotePutPrm { if p != nil { p.obj = v } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 8be9c75a0..bf6c20588 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -212,7 +212,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { // enable additional container broadcast on non-local operation // if object has TOMBSTONE or LOCK type. typ := prm.hdr.Type() - withBroadcast := !prm.common.LocalOnly() && (typ == object.TypeTombstone || typ == object.TypeLock) + withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) return &distributedTarget{ traversal: traversal{ diff --git a/pkg/services/object/put/v2/util.go b/pkg/services/object/put/v2/util.go index 758470f6c..a157a9542 100644 --- a/pkg/services/object/put/v2/util.go +++ b/pkg/services/object/put/v2/util.go @@ -5,7 +5,7 @@ import ( refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.PutRequest) (*putsvc.PutInitPrm, error) { @@ -21,7 +21,7 @@ func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.Put return new(putsvc.PutInitPrm). WithObject( - object.NewFromV2(oV2), + objectSDK.NewFromV2(oV2), ). WithRelay(s.relayRequest). WithCommonPrm(commonPrm). diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 475a31b98..c1a9a0c1c 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -56,7 +56,7 @@ func (exec *execCtx) containerID() cid.ID { return exec.prm.cnr } -func (exec *execCtx) searchFilters() object.SearchFilters { +func (exec *execCtx) searchFilters() objectSDK.SearchFilters { return exec.prm.filters } diff --git a/pkg/services/object/search/prm.go b/pkg/services/object/search/prm.go index da46dfeb6..d2918d6e7 100644 --- a/pkg/services/object/search/prm.go +++ b/pkg/services/object/search/prm.go @@ -6,7 +6,7 @@ import ( coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -18,7 +18,7 @@ type Prm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters forwarder RequestForwarder } @@ -55,6 +55,6 @@ func (p *Prm) WithContainerID(id cid.ID) { } // WithSearchFilters sets search filters. -func (p *Prm) WithSearchFilters(fs object.SearchFilters) { +func (p *Prm) WithSearchFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index cfccaede6..e971fa8e5 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -12,7 +12,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -58,7 +58,7 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre } p.WithContainerID(id) - p.WithSearchFilters(object.NewSearchFiltersFromV2(body.GetFilters())) + p.WithSearchFilters(objectSDK.NewSearchFiltersFromV2(body.GetFilters())) return p, nil } diff --git a/pkg/services/object/util/chain.go b/pkg/services/object/util/chain.go index 96dafd10e..b574d5eb6 100644 --- a/pkg/services/object/util/chain.go +++ b/pkg/services/object/util/chain.go @@ -5,7 +5,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -22,11 +22,11 @@ type HeadReceiver interface { // // If reverseDirection arg is true, then the traversal is done in reverse order. // Stop boolean result provides the ability to interrupt the traversal. -type SplitMemberHandler func(member *object.Object, reverseDirection bool) (stop bool) +type SplitMemberHandler func(member *objectSDK.Object, reverseDirection bool) (stop bool) // IterateAllSplitLeaves is an iterator over all object split-tree leaves in direct order. -func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object)) error { - return IterateSplitLeaves(r, addr, func(leaf *object.Object) bool { +func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object)) error { + return IterateSplitLeaves(r, addr, func(leaf *objectSDK.Object) bool { h(leaf) return false }) @@ -35,13 +35,13 @@ func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Obje // IterateSplitLeaves is an iterator over object split-tree leaves in direct order. // // If member handler returns true, then the iterator aborts without error. -func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object) bool) error { +func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object) bool) error { var ( reverse bool - leaves []*object.Object + leaves []*objectSDK.Object ) - if err := TraverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if err := TraverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { reverse = reverseDirection if reverse { @@ -84,9 +84,9 @@ func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) switch res := v.(type) { default: panic(fmt.Sprintf("unexpected result of %T: %T", r, v)) - case *object.Object: + case *objectSDK.Object: return h(res, false), nil - case *object.SplitInfo: + case *objectSDK.SplitInfo: link, withLink := res.Link() last, withLast := res.LastPart() @@ -108,7 +108,7 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle chain := make([]oid.ID, 0) - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { children := member.Children() if reverseDirection { @@ -122,12 +122,12 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, err } - var reverseChain []*object.Object + var reverseChain []*objectSDK.Object for i := range chain { addr.SetObject(chain[i]) - if stop, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if stop, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if !reverseDirection { return h(member, false) } @@ -147,16 +147,16 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, nil } -func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *object.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { +func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *objectSDK.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { var addr oid.Address addr.SetContainer(cnr) for last, withLast = res.LastPart(); withLast; { addr.SetObject(last) - var directChain []*object.Object + var directChain []*objectSDK.Object - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if reverseDirection { last, withLast = member.PreviousID() return h(member, true) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 46fcc9840..66a0ec7d3 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -7,7 +7,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -21,7 +21,7 @@ type Source interface { // // Tombstone MUST return (nil, nil) if requested tombstone is // missing in the storage for the provided epoch. - Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*object.Object, error) + Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*objectSDK.Object, error) } // ExpirationChecker is a tombstone source wrapper. @@ -72,7 +72,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr return false } -func (g *ExpirationChecker) handleTS(addr string, ts *object.Object, reqEpoch uint64) bool { +func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch uint64) bool { for _, atr := range ts.Attributes() { if atr.Key() == objectV2.SysAttributeExpEpoch || atr.Key() == objectV2.SysAttributeExpEpochNeoFS { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index db297b68a..a45c019eb 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -97,7 +97,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == object.TypeLock { + if typ == objectSDK.TypeLock { // all nodes of a container must store the `LOCK` objects // for correct object removal protection: // - `LOCK` objects are broadcast on their PUT requests; diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index a09957895..0ead48ef4 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -14,7 +14,7 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" @@ -27,7 +27,7 @@ func TestBuryObjectWithoutContainer(t *testing.T) { objs := []objectcore.AddressWithType{ { Address: addr, - Type: object.TypeRegular, + Type: objectSDK.TypeRegular, }, } @@ -68,7 +68,7 @@ func TestProcessObject(t *testing.T) { // - policy is used only to match the number of replicas for each index in the placement tests := []struct { desc string - objType object.Type + objType objectSDK.Type nodeCount int policy string placement [][]int @@ -128,7 +128,7 @@ func TestProcessObject(t *testing.T) { }, { desc: "lock object must be replicated to all nodes", - objType: object.TypeLock, + objType: objectSDK.TypeLock, nodeCount: 3, policy: `REP 1`, placement: [][]int{{0, 1, 2}}, @@ -176,7 +176,7 @@ func TestProcessObject(t *testing.T) { } // Object remote header - headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*object.Object, error) { + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*objectSDK.Object, error) { index := int(ni.PublicKey()[0]) if a != addr || index < 1 || index >= ti.nodeCount { t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) -- 2.45.2 From a0d51090a4620beccb5a86b0808ef085c1f84f87 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 5 Jul 2023 11:57:16 +0300 Subject: [PATCH 181/233] [#482] Enable staticcheck in foregjo actions Signed-off-by: Anton Nikiforov --- .forgejo/workflows/tests.yml | 18 ++++++++++++++++++ Makefile | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 2d44d34f8..13395a64b 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -52,3 +52,21 @@ jobs: - name: Run tests run: go test ./... -count=1 -race + + staticcheck: + name: Staticcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Install staticcheck + run: make staticcheck-install + + - name: Run staticcheck + run: make staticcheck-run diff --git a/Makefile b/Makefile index e1b719206..df53b8772 100755 --- a/Makefile +++ b/Makefile @@ -135,8 +135,12 @@ pre-commit-run: lint: @golangci-lint --timeout=5m run +# Install staticcheck +staticcheck-install: + @go install honnef.co/go/tools/cmd/staticcheck@latest + # Run staticcheck -staticcheck: +staticcheck-run: @staticcheck ./... # Run linters in Docker -- 2.45.2 From e858479a746604813130aa84edb991d0861b5f62 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Jul 2023 18:23:52 +0300 Subject: [PATCH 182/233] [#498] policer: Explicitly Rewind() iterator after finish Previously, we can continue to return `EndOfListing` infinitely. Reflect iterator reuse via Rewind() method. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/keyspaceiterator.go | 4 ++ pkg/services/policer/option.go | 1 + pkg/services/policer/policer_test.go | 89 +++++++++++++++++++++++++++- pkg/services/policer/process.go | 1 + 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go index 8991964a0..e7214aacb 100644 --- a/cmd/frostfs-node/keyspaceiterator.go +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -26,3 +26,7 @@ func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objec it.cur = res.Cursor() return res.AddressList(), nil } + +func (it *keySpaceIterator) Rewind() { + it.cur = nil +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 6f17b2947..e182c6be7 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -23,6 +23,7 @@ import ( // when the end of the key space is reached. type KeySpaceIterator interface { Next(context.Context, uint32) ([]objectcore.AddressWithType, error) + Rewind() } // RedundantCopyCallback is a callback to pass diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 0ead48ef4..ad82d2477 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -244,11 +244,95 @@ func TestProcessObject(t *testing.T) { } } +func TestIteratorContract(t *testing.T) { + addr := oidtest.Address() + objs := []objectcore.AddressWithType{{ + Address: addr, + Type: object.TypeRegular, + }} + + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + return nil + } + + pool, err := ants.NewPool(4) + require.NoError(t, err) + + it := &predefinedIterator{ + scenario: []nextResult{ + {objs, nil}, + {nil, errors.New("opaque")}, + {nil, engine.ErrEndOfListing}, + {nil, engine.ErrEndOfListing}, + {nil, errors.New("opaque")}, + {objs, engine.ErrEndOfListing}, + }, + finishCh: make(chan struct{}), + } + + p := New( + WithKeySpaceIterator(it), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + WithNodeLoader(constNodeLoader(0)), + ) + + ctx, cancel := context.WithCancel(context.Background()) + go p.Run(ctx) + + <-it.finishCh + cancel() + require.Equal(t, []string{ + "Next", + "Next", + "Next", + "Rewind", + "Next", + "Rewind", + "Next", + "Next", + "Rewind", + }, it.calls) +} + // TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) func eqAddr(a, b oid.Address) bool { return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) } +type nextResult struct { + objs []objectcore.AddressWithType + err error +} + +type predefinedIterator struct { + scenario []nextResult + finishCh chan struct{} + pos int + calls []string +} + +func (it *predefinedIterator) Next(ctx context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.pos == len(it.scenario) { + close(it.finishCh) + <-ctx.Done() + return nil, nil + } + + res := it.scenario[it.pos] + it.pos += 1 + it.calls = append(it.calls, "Next") + return res.objs, res.err +} + +func (it *predefinedIterator) Rewind() { + it.calls = append(it.calls, "Rewind") +} + // sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. type sliceKeySpaceIterator struct { objs []objectcore.AddressWithType @@ -257,7 +341,6 @@ type sliceKeySpaceIterator struct { func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { if it.cur >= len(it.objs) { - it.cur = 0 return nil, engine.ErrEndOfListing } end := it.cur + int(size) @@ -269,6 +352,10 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc return ret, nil } +func (it *sliceKeySpaceIterator) Rewind() { + it.cur = 0 +} + // containerSrcFunc is a container.Source backed by a function. type containerSrcFunc func(cid.ID) (*container.Container, error) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 39b61c8a0..cdc92ed12 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -27,6 +27,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { addrs, err := p.keySpaceIterator.Next(ctx, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { + p.keySpaceIterator.Rewind() time.Sleep(time.Second) // finished whole cycle, sleep a bit continue } -- 2.45.2 From 2310a5c7ba5577ac61837418a2de94ea20fc4c7a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Jul 2023 18:29:16 +0300 Subject: [PATCH 183/233] [#498] policer: Allow to set sleep duration between iterations Speed up tests on CI. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/option.go | 3 ++- pkg/services/policer/policer_test.go | 4 ++++ pkg/services/policer/process.go | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index e182c6be7..4194353ca 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -76,7 +76,7 @@ type cfg struct { batchSize, cacheSize uint32 - rebalanceFreq, evictDuration time.Duration + rebalanceFreq, evictDuration, sleepDuration time.Duration } func defaultCfg() *cfg { @@ -85,6 +85,7 @@ func defaultCfg() *cfg { batchSize: 10, cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB rebalanceFreq: 1 * time.Second, + sleepDuration: 1 * time.Second, evictDuration: 30 * time.Second, } } diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index ad82d2477..664e7d105 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -6,6 +6,7 @@ import ( "errors" "sort" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -279,6 +280,9 @@ func TestIteratorContract(t *testing.T) { WithBuryFunc(buryFn), WithPool(pool), WithNodeLoader(constNodeLoader(0)), + func(c *cfg) { + c.sleepDuration = time.Millisecond + }, ) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index cdc92ed12..3b54bf929 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -28,7 +28,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if err != nil { if errors.Is(err, engine.ErrEndOfListing) { p.keySpaceIterator.Rewind() - time.Sleep(time.Second) // finished whole cycle, sleep a bit + time.Sleep(p.sleepDuration) // finished whole cycle, sleep a bit continue } p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) -- 2.45.2 From 140d970a95eab5df1e5c44b0d8c2c64bb85092c3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 7 Jul 2023 18:52:40 +0300 Subject: [PATCH 184/233] [#498] policer: Fix objectSDK import Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/policer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 664e7d105..42428df23 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -249,7 +249,7 @@ func TestIteratorContract(t *testing.T) { addr := oidtest.Address() objs := []objectcore.AddressWithType{{ Address: addr, - Type: object.TypeRegular, + Type: objectSDK.TypeRegular, }} containerSrc := func(id cid.ID) (*container.Container, error) { -- 2.45.2 From 040a623d391d7879d665f698966baab9db7af219 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 7 Jul 2023 18:52:32 +0300 Subject: [PATCH 185/233] [#476] cli: Fix `object nodes` command Do not fail if client creation failed. Use external addresses to create the client too. Use public key as node ID. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 358a9d618..d2d20dc08 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -3,6 +3,7 @@ package object import ( "context" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" "strconv" @@ -223,14 +224,12 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl } } - var err error eg, egCtx := errgroup.WithContext(cmd.Context()) for _, cand := range candidates { cand := cand eg.Go(func() error { - var cli *client.Client - cli, err = createClient(egCtx, cmd, cand, pk) + cli, err := createClient(egCtx, cmd, cand, pk) if err != nil { resultMtx.Lock() defer resultMtx.Unlock() @@ -268,13 +267,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl }) } - egErr := eg.Wait() - if err != nil || egErr != nil { - if err == nil { - err = egErr - } - commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", err) - } + commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) return result } @@ -285,6 +278,7 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N addresses = append(addresses, s) return false }) + addresses = append(addresses, candidate.ExternalAddresses()...) var lastErr error for _, address := range addresses { var networkAddr network.Address @@ -336,13 +330,9 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem defer func() { commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) }() - fmt.Fprintln(w, "Netmap node\tShould contain object\tActually contains object\t") + fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") for _, n := range netmap.Nodes() { - var address string - n.IterateNetworkEndpoints(func(s string) bool { - address = s - return s != "" - }) + nodeID := hex.EncodeToString(n.PublicKey()) _, required := requiredPlacement[n.Hash()] actual, actualExists := actualPlacement[n.Hash()] actualStr := "" @@ -353,6 +343,6 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem actualStr = strconv.FormatBool(actual.value) } } - fmt.Fprintf(w, "%s\t%s\t%s\t\n", address, strconv.FormatBool(required), actualStr) + fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) } } -- 2.45.2 From 9be5d44a46bf29dcf2a21b66c14ed19030714ef9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Jul 2023 11:35:51 +0300 Subject: [PATCH 186/233] [#486] node: Update api-go and sdk-go versions Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | Bin 99007 -> 99007 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4be07aef7..9a53bb8fa 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 03e60464114b5feabac5869ee9251cc7f87ac204..31a4cb9e1691022a8766fb91c20b9c3e33d5760a 100644 GIT binary patch delta 283 zcmdnr%C^6iZG$zhtGR)Rfu)hTiIJ|6g{i5bS)!#$YEr5~hM`rklc9l8P^N3Twn>I% zVRCt;yQO)Rhf_&;N~v#9Qff+4Np^XVOIUD0vBl&DUPm$PCT+gXo4^dT#mvwIh|P5^ zEs_(H%u`d$49(0yws-`WyXZT+Bo&1gn3$#q=BDLYdPgRvTU7ZOxVvb3I2z;_X@_bj z8#`9!18vcmtRg7vgWWX!^nAVC{1lL(MW$|*8 kg?S!+h32_cVab^(c{z?iLlq|12yr#*3T@XFVw_X}05}s=wg3PC delta 283 zcmdnr%C^6iZG$zhtC@k3p^1^Xp_#6k1(2C!X=sp`oTQLpXqBE=Xi}b@ZJwl^X{lY5 z?UJaSo*$51X6jZ|7@6Z+;ackKof2hgo*SHOFu8%(Q4G6Do3HaGFuR%=Sr{4_8XFkt znx>c;C8wqtnWvg1gKWw3u*}U&t8^^Qj7qIcHn((fDff%YO7cq0&#f|u3^(ygH1o~Y z&n!*!@dVnUF`jq|H=lHAflT--b|vLXx|jXg8djGcl? l@ -- 2.45.2 From 7b76527759924bd82c5849c8c8bb451244117d98 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Jul 2023 11:36:20 +0300 Subject: [PATCH 187/233] [#486] node: Add PutSingle wrappers Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 ++ pkg/network/transport/object/grpc/service.go | 14 +++++ pkg/services/object/acl/v2/service.go | 66 +++++++++++++++++++- pkg/services/object/acl/v2/util.go | 6 ++ pkg/services/object/common.go | 8 +++ pkg/services/object/metrics.go | 16 +++++ pkg/services/object/put/v2/service.go | 6 ++ pkg/services/object/response.go | 10 +++ pkg/services/object/server.go | 1 + pkg/services/object/sign.go | 16 +++++ pkg/services/object/transport_splitter.go | 4 ++ 11 files changed, 148 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 4106f5dc1..1d99d958b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -70,6 +70,10 @@ func (s *objectSvc) Put() (objectService.PutObjectStream, error) { return s.put.Put() } +func (s *objectSvc) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return s.put.PutSingle(ctx, req) +} + func (s *objectSvc) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { return s.get.Head(ctx, req) } diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 7fa60f99c..7c6b395d5 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -110,3 +110,17 @@ func (s *Server) GetRangeHash(ctx context.Context, req *objectGRPC.GetRangeHashR return resp.ToGRPCMessage().(*objectGRPC.GetRangeHashResponse), nil } + +func (s *Server) PutSingle(ctx context.Context, req *objectGRPC.PutSingleRequest) (*objectGRPC.PutSingleResponse, error) { + putSingleReq := &object.PutSingleRequest{} + if err := putSingleReq.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.PutSingle(ctx, putSingleReq) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*objectGRPC.PutSingleResponse), nil +} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6544d78d7..c75bd326f 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,6 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -443,6 +444,65 @@ func (b Service) GetRangeHash( return b.next.GetRangeHash(ctx, request) } +func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + cnr, err := getContainerIDFromRequest(request) + if err != nil { + return nil, err + } + + idV2 := request.GetBody().GetObject().GetHeader().GetOwnerID() + if idV2 == nil { + return nil, errors.New("missing object owner") + } + + var idOwner user.ID + + err = idOwner.ReadFromV2(*idV2) + if err != nil { + return nil, fmt.Errorf("invalid object owner: %w", err) + } + + obj, err := getObjectIDFromRefObjectID(request.GetBody().GetObject().GetObjectID()) + if err != nil { + return nil, err + } + + var sTok *sessionSDK.Object + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) + if err != nil { + return nil, err + } + + bTok, err := originalBearerToken(request.GetMetaHeader()) + if err != nil { + return nil, err + } + + req := MetaWithToken{ + vheader: request.GetVerificationHeader(), + token: sTok, + bearer: bTok, + src: request, + } + + reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectPut) + if err != nil { + return nil, err + } + + reqInfo.obj = obj + + if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { + return nil, basicACLErr(reqInfo) + } + + if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } + + return b.next.PutSingle(ctx, request) +} + func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { @@ -481,7 +541,7 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } var sTok *sessionSDK.Object - sTok, err = p.readSessionToken(cnr, obj, request) + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) if err != nil { return err } @@ -515,10 +575,10 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe return p.next.Send(ctx, request) } -func (p putStreamBasicChecker) readSessionToken(cnr cid.ID, obj *oid.ID, request *objectV2.PutRequest) (*sessionSDK.Object, error) { +func readSessionToken(cnr cid.ID, obj *oid.ID, tokV2 *session.Token) (*sessionSDK.Object, error) { var sTok *sessionSDK.Object - if tokV2 := request.GetMetaHeader().GetSessionToken(); tokV2 != nil { + if tokV2 != nil { sTok = new(sessionSDK.Object) err := sTok.ReadFromV2(*tokV2) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index cd45b63fc..feda6a3cf 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -44,6 +44,8 @@ func getContainerIDFromRequest(req any) (cid.ID, error) { idV2 = v.GetBody().GetAddress().GetContainerID() case *objectV2.GetRangeHashRequest: idV2 = v.GetBody().GetAddress().GetContainerID() + case *objectV2.PutSingleRequest: + idV2 = v.GetBody().GetObject().GetHeader().GetContainerID() default: return cid.ID{}, errors.New("unknown request type") } @@ -97,6 +99,10 @@ func originalSessionToken(header *sessionV2.RequestMetaHeader) (*sessionSDK.Obje // object reference's holders. Returns an error if object ID is missing in the request. func getObjectIDFromRequestBody(body interface{ GetAddress() *refsV2.Address }) (*oid.ID, error) { idV2 := body.GetAddress().GetObjectID() + return getObjectIDFromRefObjectID(idV2) +} + +func getObjectIDFromRefObjectID(idV2 *refsV2.ObjectID) (*oid.ID, error) { if idV2 == nil { return nil, errors.New("missing object ID") } diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 5b139d8eb..0d39dce0b 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -89,3 +89,11 @@ func (x *Common) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashReq return x.nextHandler.GetRangeHash(ctx, req) } + +func (x *Common) PutSingle(ctx context.Context, req *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + if x.state.IsMaintenance() { + return nil, errMaintenance + } + + return x.nextHandler.PutSingle(ctx, req) +} diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 487374940..f972f43ae 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -76,6 +76,22 @@ func (m MetricCollector) Put() (PutObjectStream, error) { return m.next.Put() } +func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingleRequest) (*object.PutSingleResponse, error) { + if m.enabled { + t := time.Now() + + res, err := m.next.PutSingle(ctx, request) + + m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil) + if err == nil { + m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload())) + } + + return res, err + } + return m.next.PutSingle(ctx, request) +} + func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { if m.enabled { t := time.Now() diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 656f8df9c..5af62cd40 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -1,8 +1,10 @@ package putsvc import ( + "context" "fmt" + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -47,6 +49,10 @@ func (s *Service) Put() (object.PutObjectStream, error) { }, nil } +func (s *Service) PutSingle(context.Context, *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + return nil, fmt.Errorf("unimplemented") //TODO +} + func WithInternalService(v *putsvc.Service) Option { return func(c *cfg) { c.svc = v diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 81d6aaaaf..a10f26a34 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -87,6 +87,16 @@ func (s *ResponseService) Put() (PutObjectStream, error) { }, nil } +func (s *ResponseService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := s.svc.PutSingle(ctx, req) + if err != nil { + return nil, err + } + + s.respSvc.SetMeta(resp) + return resp, nil +} + func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { resp, err := s.svc.Head(ctx, req) if err != nil { diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index ccce9c4f4..73b88f233 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -41,4 +41,5 @@ type ServiceServer interface { Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) GetRange(*object.GetRangeRequest, GetObjectRangeStream) error GetRangeHash(context.Context, *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) + PutSingle(context.Context, *object.PutSingleRequest) (*object.PutSingleResponse, error) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 9d66c76ba..5b3578e29 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -120,6 +120,22 @@ func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*objec return resp.(*object.HeadResponse), nil } +func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, + func(ctx context.Context, req any) (util.ResponseMessage, error) { + return s.svc.PutSingle(ctx, req.(*object.PutSingleRequest)) + }, + func() util.ResponseMessage { + return new(object.PutSingleResponse) + }, + ) + if err != nil { + return nil, err + } + + return resp.(*object.PutSingleResponse), nil +} + func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true return s.respWriter(resp) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index a7d1c486a..2d9810cd3 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -107,6 +107,10 @@ func (c TransportSplitter) Delete(ctx context.Context, request *object.DeleteReq return c.next.Delete(ctx, request) } +func (c TransportSplitter) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return c.next.PutSingle(ctx, req) +} + func (s *rangeStreamMsgSizeCtrl) Send(resp *object.GetRangeResponse) error { body := resp.GetBody() -- 2.45.2 From fcbf90d31b7c932fda251fc4d186c07b7ccf1aba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Jul 2023 15:12:59 +0300 Subject: [PATCH 188/233] [#486] node: Add PutSingle implemetation Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 369 ++++++++++++++++++++++++++ pkg/services/object/put/v2/service.go | 4 +- 2 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 pkg/services/object/put/single.go diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go new file mode 100644 index 000000000..32c1ca73a --- /dev/null +++ b/pkg/services/object/put/single.go @@ -0,0 +1,369 @@ +package putsvc + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "hash" + "sync" + "sync/atomic" + + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +type putSingleRequestSigner struct { + req *objectAPI.PutSingleRequest + keyStorage *svcutil.KeyStorage + signer *sync.Once +} + +func (s *putSingleRequestSigner) GetRequestWithSignedHeader() (*objectAPI.PutSingleRequest, error) { + var resErr error + s.signer.Do(func() { + metaHdr := new(sessionV2.RequestMetaHeader) + meta := s.req.GetMetaHeader() + + metaHdr.SetTTL(meta.GetTTL() - 1) + metaHdr.SetOrigin(meta) + s.req.SetMetaHeader(metaHdr) + + privateKey, err := s.keyStorage.GetKey(nil) + if err != nil { + resErr = err + return + } + resErr = signature.SignServiceMessage(privateKey, s.req) + }) + return s.req, resErr +} + +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putsvc.PutSingle") + defer span.End() + + obj := objectSDK.NewFromV2(req.GetBody().GetObject()) + + if err := s.validatePutSingle(ctx, obj); err != nil { + return nil, err + } + + if err := s.saveToNodes(ctx, obj, req); err != nil { + return nil, err + } + + resp := &objectAPI.PutSingleResponse{} + resp.SetBody(&objectAPI.PutSingleResponseBody{}) + return resp, nil +} + +func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) error { + if err := s.validarePutSingleSize(obj); err != nil { + return err + } + + if err := s.validatePutSingleChecksum(obj); err != nil { + return err + } + + return s.validatePutSingleObject(ctx, obj) +} + +func (s *Service) validarePutSingleSize(obj *objectSDK.Object) error { + if uint64(len(obj.Payload())) != obj.PayloadSize() { + return ErrWrongPayloadSize + } + + maxAllowedSize := s.maxSizeSrc.MaxObjectSize() + if obj.PayloadSize() > maxAllowedSize { + return ErrExceedingMaxSize + } + + return nil +} + +func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { + cs, csSet := obj.PayloadChecksum() + if !csSet { + return errors.New("missing payload checksum") + } + + var hash hash.Hash + + switch typ := cs.Type(); typ { + default: + return fmt.Errorf("unsupported payload checksum type %v", typ) + case checksum.SHA256: + hash = sha256.New() + case checksum.TZ: + hash = tz.New() + } + + if _, err := hash.Write(obj.Payload()); err != nil { + return fmt.Errorf("could not compute payload hash: %w", err) + } + + if !bytes.Equal(hash.Sum(nil), cs.Value()) { + return fmt.Errorf("incorrect payload checksum") + } + + return nil +} + +func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { + if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { + return fmt.Errorf("coult not validate object format: %w", err) + } + + _, err := s.fmtValidator.ValidateContent(obj) + if err != nil { + return fmt.Errorf("could not validate payload content: %w", err) + } + + return nil +} + +func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest) error { + localOnly := req.GetMetaHeader().GetTTL() <= 1 + placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + if err != nil { + return err + } + traversal := &traversal{ + opts: placementOptions, + extraBroadcastEnabled: len(obj.Children()) > 0 || + (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), + mtx: sync.RWMutex{}, + mExclude: make(map[string]struct{}), + } + signer := &putSingleRequestSigner{ + req: req, + keyStorage: s.keyStorage, + signer: &sync.Once{}, + } + return s.saveAccordingToPlacement(ctx, obj, signer, traversal) +} + +func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { + var result []placement.Option + if len(copiesNumber) > 0 { + result = append(result, placement.WithCopyNumbers(copiesNumber)) + } + + cnrID, ok := obj.ContainerID() + if !ok { + return nil, errors.New("missing container ID") + } + cnrInfo, err := s.cnrSrc.Get(cnrID) + if err != nil { + return nil, fmt.Errorf("could not get container by ID: %w", err) + } + result = append(result, placement.ForContainer(cnrInfo.Value)) + + objID, ok := obj.ID() + if !ok { + return nil, errors.New("missing object ID") + } + result = append(result, placement.ForObject(objID)) + + latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) + if err != nil { + return nil, fmt.Errorf("could not get latest network map: %w", err) + } + builder := placement.NewNetworkMapBuilder(latestNetmap) + if localOnly { + result = append(result, placement.SuccessAfter(1)) + builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) + } + result = append(result, placement.UseBuilder(builder)) + return result, nil +} + +func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, traversal *traversal) error { + traverser, err := placement.NewTraverser(traversal.opts...) + if err != nil { + return fmt.Errorf("could not create object placement traverser: %w", err) + } + + var resultError atomic.Value + for { + addrs := traverser.Next() + if len(addrs) == 0 { + break + } + + if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, &resultError); stop { + break + } + } + + if !traverser.Success() { + var err errIncompletePut + err.singleErr, _ = resultError.Load().(error) + return err + } + + if traversal.submitPrimaryPlacementFinish() { + err = s.saveAccordingToPlacement(ctx, obj, signer, traversal) + if err != nil { + s.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + } + } + + return nil +} + +func (s *Service) saveToPlacementNodes(ctx context.Context, + obj *objectSDK.Object, + signer *putSingleRequestSigner, + traversal *traversal, + traverser *placement.Traverser, + nodeAddresses []placement.Node, + resultError *atomic.Value, +) bool { + wg := sync.WaitGroup{} + + for _, nodeAddress := range nodeAddresses { + nodeAddress := nodeAddress + if traversal.processed(nodeAddress) { + continue + } + + local := false + workerPool := s.remotePool + if s.netmapKeys.IsLocalKey(nodeAddress.PublicKey()) { + local = true + workerPool = s.localPool + } + + wg.Add(1) + if err := workerPool.Submit(func() { + defer wg.Done() + + err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer) + + traversal.submitProcessed(nodeAddress) + + if err != nil { + resultError.Store(err) + svcutil.LogServiceError(s.log, "PUT", nodeAddress.Addresses(), err) + return + } + + traverser.SubmitSuccess() + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(s.log, "PUT", err) + return true + } + } + + wg.Wait() + + return false +} + +func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, signer *putSingleRequestSigner) error { + if nodeDesc.local { + return s.localStore.Put(ctx, obj) + } + + var info client.NodeInfo + + client.NodeInfoFromNetmapElement(&info, nodeDesc.info) + + c, err := s.clientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + return s.redirectPutSingleRequest(ctx, signer, obj, info, c) +} + +func (s *Service) redirectPutSingleRequest(ctx context.Context, + signer *putSingleRequestSigner, + obj *objectSDK.Object, + info client.NodeInfo, + c client.MultiAddressClient) error { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest") + defer span.End() + + var req *objectAPI.PutSingleRequest + var firstErr error + req, firstErr = signer.GetRequestWithSignedHeader() + if firstErr != nil { + return firstErr + } + + info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest.IterateAddresses", + trace.WithAttributes( + attribute.String("address", addr.String()), + )) + defer span.End() + + var err error + + defer func() { + if err != nil { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + s.log.Warn("failed to redirect PutSingle request", + zap.Error(err), + zap.Stringer("address", addr), + zap.Stringer("object_id", objID), + zap.Stringer("container_id", cnrID), + ) + } + + stop = err == nil + if stop || firstErr == nil { + firstErr = err + } + }() + + var resp *objectAPI.PutSingleResponse + + err = c.RawForAddress(ctx, addr, func(cli *rawclient.Client) error { + var e error + resp, e = rpc.PutSingleObject(cli, req, rawclient.WithContext(ctx)) + return e + }) + if err != nil { + err = fmt.Errorf("failed to execute request: %w", err) + return + } + + if err = internal.VerifyResponseKeyV2(info.PublicKey(), resp); err != nil { + return + } + + err = signature.VerifyServiceMessage(resp) + if err != nil { + err = fmt.Errorf("response verification failed: %w", err) + } + + return + }) + + return firstErr +} diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 5af62cd40..78655edc7 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -49,8 +49,8 @@ func (s *Service) Put() (object.PutObjectStream, error) { }, nil } -func (s *Service) PutSingle(context.Context, *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { - return nil, fmt.Errorf("unimplemented") //TODO +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + return s.svc.PutSingle(ctx, req) } func WithInternalService(v *putsvc.Service) Option { -- 2.45.2 From a65e26878b287a304713845d23a9c4ae1516c8aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Jul 2023 15:42:31 +0300 Subject: [PATCH 189/233] [#486] put service: Fix error typo Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 2 +- pkg/services/object/put/validation.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 32c1ca73a..200830e15 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -131,7 +131,7 @@ func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("coult not validate object format: %w", err) + return fmt.Errorf("coud not validate object format: %w", err) } _, err := s.fmtValidator.ValidateContent(obj) diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 406304422..d4b08a038 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -48,7 +48,7 @@ var ( func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { if err := t.fmt.Validate(ctx, obj, true); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } return t.nextTarget.WriteHeader(ctx, obj) @@ -93,7 +93,7 @@ func (t *validatingPreparedTarget) WriteHeader(ctx context.Context, obj *objectS t.checksum = cs.Value() if err := t.fmt.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } err := t.nextTarget.WriteHeader(ctx, obj) -- 2.45.2 From c42db4e761790d7221814e95df7c74cd7ae46036 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 12:43:48 +0300 Subject: [PATCH 190/233] [#390] cli: Support more tree service API calls in CLI * Support healthcheck API call * Support move API call * Support remove API call * Support getSubtree API call * Support getOpLog API call Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add_by_path.go | 1 - cmd/frostfs-cli/modules/tree/get_by_path.go | 10 +- cmd/frostfs-cli/modules/tree/get_op_log.go | 83 +++++++++++++++ cmd/frostfs-cli/modules/tree/healthcheck.go | 42 ++++++++ cmd/frostfs-cli/modules/tree/move.go | 107 ++++++++++++++++++++ cmd/frostfs-cli/modules/tree/remove.go | 74 ++++++++++++++ cmd/frostfs-cli/modules/tree/root.go | 18 +++- cmd/frostfs-cli/modules/tree/subtree.go | 101 ++++++++++++++++++ 8 files changed, 430 insertions(+), 6 deletions(-) create mode 100644 cmd/frostfs-cli/modules/tree/get_op_log.go create mode 100644 cmd/frostfs-cli/modules/tree/healthcheck.go create mode 100644 cmd/frostfs-cli/modules/tree/move.go create mode 100644 cmd/frostfs-cli/modules/tree/remove.go create mode 100644 cmd/frostfs-cli/modules/tree/subtree.go diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 90f4a6364..26e3dddfc 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -62,7 +62,6 @@ func addByPath(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "meta data parsing: %w", err) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) req := new(tree.AddByPathRequest) req.Body = &tree.AddByPathRequest_Body{ diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 38ad2bff5..c76027938 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -60,7 +60,11 @@ func getByPath(cmd *cobra.Command, _ []string) { latestOnly, _ := cmd.Flags().GetBool(latestOnlyFlagKey) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } req := new(tree.GetNodeByPathRequest) req.Body = &tree.GetNodeByPathRequest_Body{ @@ -71,9 +75,7 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - } - if btok := common.ReadBearerToken(cmd, bearerFlagKey); btok != nil { - req.Body.BearerToken = btok.Marshal() + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go new file mode 100644 index 000000000..6efa76133 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -0,0 +1,83 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getOpLogCmd = &cobra.Command{ + Use: "get-op-log", + Short: "Get logged operations starting with some height", + Run: getOpLog, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetOpLogCmd() { + commonflags.Init(getOpLogCmd) + initCTID(getOpLogCmd) + + ff := getOpLogCmd.Flags() + ff.Uint64(heightFlagKey, 0, "Height to start with") + ff.Uint64(countFlagKey, 10, "Logged operations count") + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getOpLog(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + + cidRaw, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidRaw) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + height, _ := cmd.Flags().GetUint64(heightFlagKey) + count, _ := cmd.Flags().GetUint64(countFlagKey) + + req := &tree.GetOpLogRequest{ + Body: &tree.GetOpLogRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + Height: height, + Count: count, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetOpLog(ctx, req) + commonCmd.ExitOnErr(cmd, "get op log: %w", err) + + opLogResp, err := resp.Recv() + for ; err == nil; opLogResp, err = resp.Recv() { + o := opLogResp.GetBody().GetOperation() + + cmd.Println("Parent ID: ", o.GetParentId()) + + cmd.Println("\tChild ID: ", o.GetChildId()) + cmd.Printf("\tMeta: %s\n", o.GetMeta()) + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "get op log response stream: %w", err) + } +} diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go new file mode 100644 index 000000000..46d4c3409 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -0,0 +1,42 @@ +package tree + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "github.com/spf13/cobra" +) + +var healthcheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Check tree service availability", + Run: healthcheck, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initHealthcheckCmd() { + commonflags.Init(healthcheckCmd) + ff := healthcheckCmd.Flags() + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func healthcheck(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + req := &tree.HealthcheckRequest{ + Body: &tree.HealthcheckRequest_Body{}, + } + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Healthcheck(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go new file mode 100644 index 000000000..b91769185 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -0,0 +1,107 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var moveCmd = &cobra.Command{ + Use: "move", + Short: "Move node", + Run: move, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initMoveCmd() { + commonflags.Init(moveCmd) + initCTID(moveCmd) + + ff := moveCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + ff.Uint64(parentIDFlagKey, 0, "Parent ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + _ = getSubtreeCmd.MarkFlagRequired(parentIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func move(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + pid, _ := cmd.Flags().GetUint64(parentIDFlagKey) + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + subTreeReq := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: nid, + Depth: 1, + BearerToken: bt, + }, + } + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(subTreeReq, pk)) + resp, err := cli.GetSubTree(ctx, subTreeReq) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + var meta []*tree.KeyValue + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + meta = subtreeResp.GetBody().GetMeta() + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + } + var metaErr error + if len(meta) == 0 { + metaErr = errors.New("no meta for given node ID") + } + commonCmd.ExitOnErr(cmd, "unexpected rpc call result: %w", metaErr) + + req := &tree.MoveRequest{ + Body: &tree.MoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + ParentId: pid, + NodeId: nid, + Meta: meta, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Move(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go new file mode 100644 index 000000000..7002ac9a4 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -0,0 +1,74 @@ +package tree + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove node", + Run: remove, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initRemoveCmd() { + commonflags.Init(removeCmd) + initCTID(removeCmd) + + ff := removeCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func remove(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := &tree.RemoveRequest{ + Body: &tree.RemoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + NodeId: nid, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Remove(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index b5de0dd6f..701a78f2a 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -15,16 +15,28 @@ func init() { Cmd.AddCommand(getByPathCmd) Cmd.AddCommand(addByPathCmd) Cmd.AddCommand(listCmd) + Cmd.AddCommand(healthcheckCmd) + Cmd.AddCommand(moveCmd) + Cmd.AddCommand(removeCmd) + Cmd.AddCommand(getSubtreeCmd) + Cmd.AddCommand(getOpLogCmd) initAddCmd() initGetByPathCmd() initAddByPathCmd() initListCmd() + initHealthcheckCmd() + initMoveCmd() + initRemoveCmd() + initGetSubtreeCmd() + initGetOpLogCmd() } const ( treeIDFlagKey = "tid" parentIDFlagKey = "pid" + nodeIDFlagKey = "nid" + rootIDFlagKey = "root" metaFlagKey = "meta" @@ -34,6 +46,10 @@ const ( latestOnlyFlagKey = "latest" bearerFlagKey = "bearer" + + heightFlagKey = "height" + countFlagKey = "count" + depthFlagKey = "depth" ) func initCTID(cmd *cobra.Command) { @@ -45,5 +61,5 @@ func initCTID(cmd *cobra.Command) { ff.String(treeIDFlagKey, "", "Tree ID") _ = cmd.MarkFlagRequired(treeIDFlagKey) - ff.StringP(bearerFlagKey, "", "", "Path to bearer token") + ff.String(bearerFlagKey, "", "Path to bearer token") } diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go new file mode 100644 index 000000000..f6dcf85ff --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -0,0 +1,101 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getSubtreeCmd = &cobra.Command{ + Use: "get-subtree", + Short: "Get subtree", + Run: getSubTree, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetSubtreeCmd() { + commonflags.Init(getSubtreeCmd) + initCTID(getSubtreeCmd) + + ff := getSubtreeCmd.Flags() + ff.Uint64(rootIDFlagKey, 0, "Root ID to traverse from.") + ff.Uint32(depthFlagKey, 10, "Traversal depth.") + + _ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag) + _ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getSubTree(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + rid, _ := cmd.Flags().GetUint64(rootIDFlagKey) + + depth, _ := cmd.Flags().GetUint32(depthFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + req := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: rid, + Depth: depth, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetSubTree(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + b := subtreeResp.GetBody() + + cmd.Printf("Node ID: %d\n", b.GetNodeId()) + + cmd.Println("\tParent ID: ", b.GetParentId()) + cmd.Println("\tTimestamp: ", b.GetTimestamp()) + + if meta := b.GetMeta(); len(meta) > 0 { + cmd.Println("\tMeta pairs: ") + for _, kv := range meta { + cmd.Printf("\t\t%s: %s\n", kv.GetKey(), string(kv.GetValue())) + } + } + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + } +} -- 2.45.2 From 676a3efa79e136fc95ced5c8263c16032dd64f7b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 12:36:33 +0300 Subject: [PATCH 191/233] [#390] cli: Make tree commands errors and messages more verbose Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 6 +++--- cmd/frostfs-cli/modules/tree/add_by_path.go | 6 +++--- cmd/frostfs-cli/modules/tree/get_by_path.go | 6 +++--- cmd/frostfs-cli/modules/tree/healthcheck.go | 9 +++++---- cmd/frostfs-cli/modules/tree/list.go | 6 +++--- cmd/frostfs-cli/modules/tree/move.go | 12 ++++++------ cmd/frostfs-cli/modules/tree/remove.go | 8 ++++---- cmd/frostfs-cli/modules/tree/subtree.go | 6 +++--- 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index ea9245271..28718687d 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -50,7 +50,7 @@ func add(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -64,10 +64,10 @@ func add(cmd *cobra.Command, _ []string) { BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.Add(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to cal add: %w", err) cmd.Println("Node ID: ", resp.Body.NodeId) } diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 26e3dddfc..4661853a5 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -53,7 +53,7 @@ func addByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -74,10 +74,10 @@ func addByPath(cmd *cobra.Command, _ []string) { BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.AddByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to addByPath %w", err) cmd.Printf("Parent ID: %d\n", resp.GetBody().GetParentId()) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index c76027938..f239066cd 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -53,7 +53,7 @@ func getByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -78,10 +78,10 @@ func getByPath(cmd *cobra.Command, _ []string) { BearerToken: bt, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.GetNodeByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call getNodeByPath: %w", err) nn := resp.GetBody().GetNodes() if len(nn) == 0 { diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go index 46d4c3409..f0506467e 100644 --- a/cmd/frostfs-cli/modules/tree/healthcheck.go +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -1,6 +1,7 @@ package tree import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -28,15 +29,15 @@ func healthcheck(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) req := &tree.HealthcheckRequest{ Body: &tree.HealthcheckRequest_Body{}, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Healthcheck(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call healthcheck: %w", err) - cmd.Println("Success.") + common.PrintVerbose(cmd, "Successful healthcheck invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/list.go b/cmd/frostfs-cli/modules/tree/list.go index 8e4d2bd4c..a25d066d5 100644 --- a/cmd/frostfs-cli/modules/tree/list.go +++ b/cmd/frostfs-cli/modules/tree/list.go @@ -41,7 +41,7 @@ func list(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -52,10 +52,10 @@ func list(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.TreeList(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call treeList %w", err) for _, treeID := range resp.GetBody().GetIds() { cmd.Println(treeID) diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index b91769185..84b2fb80e 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -48,7 +48,7 @@ func move(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -71,7 +71,7 @@ func move(cmd *cobra.Command, _ []string) { BearerToken: bt, }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(subTreeReq, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(subTreeReq, pk)) resp, err := cli.GetSubTree(ctx, subTreeReq) commonCmd.ExitOnErr(cmd, "rpc call: %w", err) @@ -81,7 +81,7 @@ func move(cmd *cobra.Command, _ []string) { meta = subtreeResp.GetBody().GetMeta() } if !errors.Is(err, io.EOF) { - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to read getSubTree response stream: %w", err) } var metaErr error if len(meta) == 0 { @@ -99,9 +99,9 @@ func move(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Move(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) - cmd.Println("Success.") + commonCmd.ExitOnErr(cmd, "failed to call move: %w", err) + common.PrintVerbose(cmd, "Successful move invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go index 7002ac9a4..74e9d9749 100644 --- a/cmd/frostfs-cli/modules/tree/remove.go +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -44,7 +44,7 @@ func remove(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -66,9 +66,9 @@ func remove(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Remove(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) - cmd.Println("Success.") + commonCmd.ExitOnErr(cmd, "failed to call remove: %w", err) + common.PrintVerbose(cmd, "Successful remove invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index f6dcf85ff..64cb351ec 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -48,7 +48,7 @@ func getSubTree(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -74,10 +74,10 @@ func getSubTree(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.GetSubTree(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call getSubTree: %w", err) subtreeResp, err := resp.Recv() for ; err == nil; subtreeResp, err = resp.Recv() { -- 2.45.2 From 0754e6e65403b28cc73b1115f309939b2dd24c44 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 13:31:49 +0300 Subject: [PATCH 192/233] [#390] cli: Fix bearer token reading for tree subcommands Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 7 ++++++- cmd/frostfs-cli/modules/tree/add_by_path.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 28718687d..4ac1ed198 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -55,13 +55,18 @@ func add(cmd *cobra.Command, _ []string) { rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := new(tree.AddRequest) req.Body = &tree.AddRequest_Body{ ContainerId: rawCID, TreeId: tid, ParentId: pid, Meta: meta, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 4661853a5..ea815dbfe 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -63,6 +63,11 @@ func addByPath(cmd *cobra.Command, _ []string) { path, _ := cmd.Flags().GetString(pathFlagKey) + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := new(tree.AddByPathRequest) req.Body = &tree.AddByPathRequest_Body{ ContainerId: rawCID, @@ -71,7 +76,7 @@ func addByPath(cmd *cobra.Command, _ []string) { // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) -- 2.45.2 From 80481c015c236fac0dfb7a600952291f92738858 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jul 2023 17:33:13 +0300 Subject: [PATCH 193/233] [#509] putsvc: Omit AccessIdentifiers from iteratePlacement() Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 51678fa7b..c8294c0b8 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -146,7 +146,14 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent t.traversal.extraBroadcastEnabled = true } - return t.iteratePlacement(ctx) + if err := t.iteratePlacement(ctx); err != nil { + return nil, err + } + + id, _ := t.obj.ID() + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil } func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { @@ -164,14 +171,14 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error return nil } -func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() traverser, err := placement.NewTraverser( append(t.traversal.opts, placement.ForObject(id))..., ) if err != nil { - return nil, fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) + return fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) } resErr := &atomic.Value{} @@ -190,23 +197,19 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer. if !traverser.Success() { var err errIncompletePut err.singleErr, _ = resErr.Load().(error) - return nil, err + return err } // perform additional container broadcast if needed if t.traversal.submitPrimaryPlacementFinish() { - _, err = t.iteratePlacement(ctx) + err = t.iteratePlacement(ctx) if err != nil { t.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } - id, _ = t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { -- 2.45.2 From 7da284f3e84d596fca56e93947c98bc74e579706 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jul 2023 17:32:00 +0300 Subject: [PATCH 194/233] [#509] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | Bin 99007 -> 99007 bytes pkg/services/object/put/distributed.go | 31 +++++++++++++++---------- pkg/services/object/put/streamer.go | 6 ++--- pkg/services/object/put/validation.go | 4 ++-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 9a53bb8fa..e2735b223 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 31a4cb9e1691022a8766fb91c20b9c3e33d5760a..ddbc776336c5b9c02464f6cba41387ea84b1f6d2 100644 GIT binary patch delta 179 zcmdnr%C^6iZG(lNi=m;RiIJhPsjj7^MOvz1qIqJHv7thSp;bVzM_5Uvi*Z@GkzYts zO1M*Kl|gu6j$ycmiBVp#bAYxT3RG0CYh(Em>HUxDP$O0c?6fc=sUY46@?a*8g?S!+h32_cVab^(c{z?iGZiM+2yr#*3T@XF IVw_X}0Jo?%hX4Qo diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index c8294c0b8..408ee099c 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -135,18 +135,7 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent t.obj.SetPayload(t.payload.Data) - var err error - - if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { - return nil, fmt.Errorf("(%T) could not validate payload content: %w", t, err) - } - - if len(t.obj.Children()) > 0 { - // enabling extra broadcast for linking objects - t.traversal.extraBroadcastEnabled = true - } - - if err := t.iteratePlacement(ctx); err != nil { + if err := t.WriteObject(ctx, t.obj); err != nil { return nil, err } @@ -156,6 +145,24 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent }, nil } +// WriteObject implements the transformer.ObjectWriter interface. +func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + t.obj = obj + + var err error + + if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { + return fmt.Errorf("(%T) could not validate payload content: %w", t, err) + } + + if len(t.obj.Children()) > 0 { + // enabling extra broadcast for linking objects + t.traversal.extraBroadcastEnabled = true + } + + return t.iteratePlacement(ctx) +} + func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { if !node.local && t.relay != nil { return t.relay(ctx, node) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index bf6c20588..2dccafa47 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -22,7 +22,7 @@ type Streamer struct { sessionKey *ecdsa.PrivateKey - target transformer.ObjectTarget + target transformer.ChunkedObjectWriter relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error @@ -129,7 +129,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ Key: sessionKey, - NextTargetInit: func() transformer.ObjectTarget { return p.newCommonTarget(prm) }, + NextTargetInit: func() transformer.ObjectWriter { return p.newCommonTarget(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), @@ -192,7 +192,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { return nil } -func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { +func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index d4b08a038..c2b078ef5 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -17,14 +17,14 @@ import ( // validatingTarget validates unprepared object format and content (streaming PUT case). type validatingTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator } // validatingPreparedTarget validates prepared object format and content. type validatingPreparedTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator -- 2.45.2 From 61541eaec2c0feab381b57dc512a6380395c7b2f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 10:05:09 +0300 Subject: [PATCH 195/233] [#294] aclsvc: Refactor checker constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 11 +++--- pkg/services/object/acl/acl.go | 55 ++++++----------------------- pkg/services/object/acl/acl_test.go | 11 +++--- 3 files changed, 20 insertions(+), 57 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1d99d958b..45b1089a7 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -429,12 +429,11 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Se ), v2.WithNextService(splitSvc), v2.WithEACLChecker( - acl.NewChecker(new(acl.CheckerPrm). - SetNetmapState(c.cfgNetmap.state). - SetEACLSource(c.cfgObject.eaclSource). - SetValidator(eaclSDK.NewValidator()). - SetLocalStorage(ls), - ), + acl.NewChecker( + c.cfgNetmap.state, + c.cfgObject.eaclSource, + eaclSDK.NewValidator(), + ls), ), ) } diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index 351b4ad3b..cfe49396e 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -21,35 +21,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -// CheckerPrm groups parameters for Checker -// constructor. -type CheckerPrm struct { - eaclSrc container.EACLSource - validator *eaclSDK.Validator - localStorage *engine.StorageEngine - state netmap.State -} - -func (c *CheckerPrm) SetEACLSource(v container.EACLSource) *CheckerPrm { - c.eaclSrc = v - return c -} - -func (c *CheckerPrm) SetValidator(v *eaclSDK.Validator) *CheckerPrm { - c.validator = v - return c -} - -func (c *CheckerPrm) SetLocalStorage(v *engine.StorageEngine) *CheckerPrm { - c.localStorage = v - return c -} - -func (c *CheckerPrm) SetNetmapState(v netmap.State) *CheckerPrm { - c.state = v - return c -} - // Checker implements v2.ACLChecker interfaces and provides // ACL/eACL validation functionality. type Checker struct { @@ -71,23 +42,17 @@ var ( // NewChecker creates Checker. // Panics if at least one of the parameter is nil. -func NewChecker(prm *CheckerPrm) *Checker { - panicOnNil := func(fieldName string, field any) { - if field == nil { - panic(fmt.Sprintf("incorrect field %s (%T): %v", fieldName, field, field)) - } - } - - panicOnNil("EACLSource", prm.eaclSrc) - panicOnNil("EACLValidator", prm.validator) - panicOnNil("LocalStorageEngine", prm.localStorage) - panicOnNil("NetmapState", prm.state) - +func NewChecker( + state netmap.State, + eaclSrc container.EACLSource, + validator *eaclSDK.Validator, + localStorage *engine.StorageEngine, +) *Checker { return &Checker{ - eaclSrc: prm.eaclSrc, - validator: prm.validator, - localStorage: prm.localStorage, - state: prm.state, + eaclSrc: eaclSrc, + validator: validator, + localStorage: localStorage, + state: state, } } diff --git a/pkg/services/object/acl/acl_test.go b/pkg/services/object/acl/acl_test.go index d3ad1e6fd..b9b82dac8 100644 --- a/pkg/services/object/acl/acl_test.go +++ b/pkg/services/object/acl/acl_test.go @@ -27,12 +27,11 @@ func (e emptyNetmapState) CurrentEpoch() uint64 { } func TestStickyCheck(t *testing.T) { - checker := NewChecker(new(CheckerPrm). - SetLocalStorage(&engine.StorageEngine{}). - SetValidator(eaclSDK.NewValidator()). - SetEACLSource(emptyEACLSource{}). - SetNetmapState(emptyNetmapState{}), - ) + checker := NewChecker( + emptyNetmapState{}, + emptyEACLSource{}, + eaclSDK.NewValidator(), + &engine.StorageEngine{}) t.Run("system role", func(t *testing.T) { var info v2.RequestInfo -- 2.45.2 From 18d8898b00d94b33fd2c3bddee83dbadbd6ce15b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:28:36 +0300 Subject: [PATCH 196/233] [#294] aclsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/acl.go | 36 +++++++++-------- pkg/services/object/acl/eacl/v2/eacl_test.go | 6 +-- pkg/services/object/acl/eacl/v2/headers.go | 20 ++++------ pkg/services/object/acl/eacl/v2/localstore.go | 22 ----------- pkg/services/object/acl/eacl/v2/opts.go | 39 ------------------- pkg/services/object/acl/eacl/v2/xheader.go | 10 ++++- 6 files changed, 39 insertions(+), 94 deletions(-) delete mode 100644 pkg/services/object/acl/eacl/v2/localstore.go diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index cfe49396e..921545c8b 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -1,10 +1,12 @@ package acl import ( + "context" "crypto/ecdsa" "crypto/elliptic" "errors" "fmt" + "io" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -17,6 +19,8 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -30,6 +34,18 @@ type Checker struct { state netmap.State } +type localStorage struct { + ls *engine.StorageEngine +} + +func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + if s.ls == nil { + return nil, io.ErrUnexpectedEOF + } + + return engine.Head(ctx, s.ls, addr) +} + // Various EACL check errors. var ( errEACLDeniedByRule = errors.New("denied by rule") @@ -158,26 +174,14 @@ func getRole(reqInfo v2.RequestInfo) eaclSDK.Role { } func (c *Checker) getHeaderSource(cnr cid.ID, msg any, reqInfo v2.RequestInfo) (eaclSDK.TypedHeaderSource, error) { - hdrSrcOpts := make([]eaclV2.Option, 0, 3) - - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithLocalObjectStorage(c.localStorage), - eaclV2.WithCID(cnr), - eaclV2.WithOID(reqInfo.ObjectID()), - ) - + var xHeaderSource eaclV2.XHeaderSource if req, ok := msg.(eaclV2.Request); ok { - hdrSrcOpts = append(hdrSrcOpts, eaclV2.WithServiceRequest(req)) + xHeaderSource = eaclV2.NewRequestXHeaderSource(req) } else { - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithServiceResponse( - msg.(eaclV2.Response), - reqInfo.Request().(eaclV2.Request), - ), - ) + xHeaderSource = eaclV2.NewResponseXHeaderSource(msg.(eaclV2.Response), reqInfo.Request().(eaclV2.Request)) } - hdrSrc, err := eaclV2.NewMessageHeaderSource(hdrSrcOpts...) + hdrSrc, err := eaclV2.NewMessageHeaderSource(&localStorage{ls: c.localStorage}, xHeaderSource, cnr, eaclV2.WithOID(reqInfo.ObjectID())) if err != nil { return nil, fmt.Errorf("can't parse headers: %w", err) } diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index 6f879123b..023b99239 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -103,9 +103,9 @@ func TestHeadRequest(t *testing.T) { newSource := func(t *testing.T) eaclSDK.TypedHeaderSource { hdrSrc, err := NewMessageHeaderSource( - WithObjectStorage(lStorage), - WithServiceRequest(req), - WithCID(addr.Container()), + lStorage, + NewRequestXHeaderSource(req), + addr.Container(), WithOID(&id)) require.NoError(t, err) return hdrSrc diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index c7a1b7729..7408c96e9 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -21,7 +21,7 @@ type Option func(*cfg) type cfg struct { storage ObjectStorage - msg xHeaderSource + msg XHeaderSource cnr cid.ID obj *oid.ID @@ -46,14 +46,12 @@ type headerSource struct { incompleteObjectHeaders bool } -func defaultCfg() *cfg { - return &cfg{ - storage: new(localStorage), +func NewMessageHeaderSource(os ObjectStorage, xhs XHeaderSource, cnrID cid.ID, opts ...Option) (eaclSDK.TypedHeaderSource, error) { + cfg := &cfg{ + storage: os, + cnr: cnrID, + msg: xhs, } -} - -func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { - cfg := defaultCfg() for i := range opts { opts[i](cfg) @@ -70,7 +68,7 @@ func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { return nil, err } - res.requestHeaders = requestHeaders(cfg.msg) + res.requestHeaders = cfg.msg.GetXHeaders() return res, nil } @@ -96,10 +94,6 @@ func (x xHeader) Value() string { return (*session.XHeader)(&x).GetValue() } -func requestHeaders(msg xHeaderSource) []eaclSDK.Header { - return msg.GetXHeaders() -} - var errMissingOID = errors.New("object ID is missing") func (h *cfg) readObjectHeaders(dst *headerSource) error { diff --git a/pkg/services/object/acl/eacl/v2/localstore.go b/pkg/services/object/acl/eacl/v2/localstore.go deleted file mode 100644 index 0f23e9881..000000000 --- a/pkg/services/object/acl/eacl/v2/localstore.go +++ /dev/null @@ -1,22 +0,0 @@ -package v2 - -import ( - "context" - "io" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type localStorage struct { - ls *engine.StorageEngine -} - -func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - if s.ls == nil { - return nil, io.ErrUnexpectedEOF - } - - return engine.Head(ctx, s.ls, addr) -} diff --git a/pkg/services/object/acl/eacl/v2/opts.go b/pkg/services/object/acl/eacl/v2/opts.go index 7657e8780..d91a21c75 100644 --- a/pkg/services/object/acl/eacl/v2/opts.go +++ b/pkg/services/object/acl/eacl/v2/opts.go @@ -1,48 +1,9 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.storage = v - } -} - -func WithLocalObjectStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.storage = &localStorage{ - ls: v, - } - } -} - -func WithServiceRequest(v Request) Option { - return func(c *cfg) { - c.msg = requestXHeaderSource{ - req: v, - } - } -} - -func WithServiceResponse(resp Response, req Request) Option { - return func(c *cfg) { - c.msg = responseXHeaderSource{ - resp: resp, - req: req, - } - } -} - -func WithCID(v cid.ID) Option { - return func(c *cfg) { - c.cnr = v - } -} - func WithOID(v *oid.ID) Option { return func(c *cfg) { c.obj = v diff --git a/pkg/services/object/acl/eacl/v2/xheader.go b/pkg/services/object/acl/eacl/v2/xheader.go index 246714af7..c1fdea9d8 100644 --- a/pkg/services/object/acl/eacl/v2/xheader.go +++ b/pkg/services/object/acl/eacl/v2/xheader.go @@ -5,7 +5,7 @@ import ( eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" ) -type xHeaderSource interface { +type XHeaderSource interface { GetXHeaders() []eaclSDK.Header } @@ -13,12 +13,20 @@ type requestXHeaderSource struct { req Request } +func NewRequestXHeaderSource(req Request) XHeaderSource { + return requestXHeaderSource{req: req} +} + type responseXHeaderSource struct { resp Response req Request } +func NewResponseXHeaderSource(resp Response, req Request) XHeaderSource { + return responseXHeaderSource{resp: resp, req: req} +} + func (s requestXHeaderSource) GetXHeaders() []eaclSDK.Header { ln := 0 -- 2.45.2 From 70a108198884211b35f10b1a196eefe3234e56ef Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:44:37 +0300 Subject: [PATCH 197/233] [#294] aclsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 22 +++++++-------- pkg/services/object/acl/v2/opts.go | 39 --------------------------- pkg/services/object/acl/v2/service.go | 34 ++++++++++------------- 3 files changed, 23 insertions(+), 72 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 45b1089a7..da7458170 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -421,20 +421,16 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Se irFetcher := createInnerRingFetcher(c) return v2.New( + splitSvc, + c.netMapSource, + newCachedIRFetcher(irFetcher), + acl.NewChecker( + c.cfgNetmap.state, + c.cfgObject.eaclSource, + eaclSDK.NewValidator(), + ls), + c.cfgObject.cnrSource, v2.WithLogger(c.log), - v2.WithIRFetcher(newCachedIRFetcher(irFetcher)), - v2.WithNetmapSource(c.netMapSource), - v2.WithContainerSource( - c.cfgObject.cnrSource, - ), - v2.WithNextService(splitSvc), - v2.WithEACLChecker( - acl.NewChecker( - c.cfgNetmap.state, - c.cfgObject.eaclSource, - eaclSDK.NewValidator(), - ls), - ), ) } diff --git a/pkg/services/object/acl/v2/opts.go b/pkg/services/object/acl/v2/opts.go index 7e937da06..15fcce884 100644 --- a/pkg/services/object/acl/v2/opts.go +++ b/pkg/services/object/acl/v2/opts.go @@ -1,9 +1,6 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -13,39 +10,3 @@ func WithLogger(v *logger.Logger) Option { c.log = v } } - -// WithNetmapSource return option to set -// netmap source. -func WithNetmapSource(v netmap.Source) Option { - return func(c *cfg) { - c.nm = v - } -} - -// WithContainerSource returns option to set container source. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.containers = v - } -} - -// WithNextService returns option to set next object service. -func WithNextService(v objectSvc.ServiceServer) Option { - return func(c *cfg) { - c.next = v - } -} - -// WithEACLChecker returns option to set eACL checker. -func WithEACLChecker(v ACLChecker) Option { - return func(c *cfg) { - c.checker = v - } -} - -// WithIRFetcher returns option to set inner ring fetcher. -func WithIRFetcher(v InnerRingFetcher) Option { - return func(c *cfg) { - c.irFetcher = v - } -} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index c75bd326f..412708b1c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -73,32 +73,26 @@ type cfg struct { next object.ServiceServer } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New is a constructor for object ACL checking service. -func New(opts ...Option) Service { - cfg := defaultCfg() +func New(next object.ServiceServer, + nm netmap.Source, + irf InnerRingFetcher, + acl ACLChecker, + cs container.Source, + opts ...Option) Service { + cfg := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + next: next, + nm: nm, + irFetcher: irf, + checker: acl, + containers: cs, + } for i := range opts { opts[i](cfg) } - panicOnNil := func(v any, name string) { - if v == nil { - panic(fmt.Sprintf("ACL service: %s is nil", name)) - } - } - - panicOnNil(cfg.next, "next Service") - panicOnNil(cfg.nm, "netmap client") - panicOnNil(cfg.irFetcher, "inner Ring fetcher") - panicOnNil(cfg.checker, "acl checker") - panicOnNil(cfg.containers, "container source") - return Service{ cfg: cfg, c: senderClassifier{ -- 2.45.2 From a476d8285abd39cc7b74284ae8ae75fed05a5ed7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:53:18 +0300 Subject: [PATCH 198/233] [#294] deletesvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +--- pkg/services/object/delete/v2/service.go | 23 ++--------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index da7458170..3f0db1461 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -397,9 +397,7 @@ func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Servi } func createDeleteServiceV2(sDelete *deletesvc.Service) *deletesvcV2.Service { - return deletesvcV2.NewService( - deletesvcV2.WithInternalService(sDelete), - ) + return deletesvcV2.NewService(sDelete) } func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Service, diff --git a/pkg/services/object/delete/v2/service.go b/pkg/services/object/delete/v2/service.go index 51759c5df..10dcd0e87 100644 --- a/pkg/services/object/delete/v2/service.go +++ b/pkg/services/object/delete/v2/service.go @@ -9,26 +9,13 @@ import ( // Service implements Delete operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *deletesvc.Service } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *deletesvc.Service) *Service { return &Service{ - cfg: c, + svc: svc, } } @@ -51,9 +38,3 @@ func (s *Service) Delete(ctx context.Context, req *objectV2.DeleteRequest) (*obj return resp, nil } - -func WithInternalService(v *deletesvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} -- 2.45.2 From 1420b8b9eae4f315c3bb1701589b1591718b6bba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:04:00 +0300 Subject: [PATCH 199/233] [#294] deletesvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 14 +++---- pkg/services/object/delete/service.go | 58 +++++++-------------------- 2 files changed, 21 insertions(+), 51 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 3f0db1461..1f763dd9a 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -382,17 +382,17 @@ func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *gets func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Service, sSearch *searchsvc.Service, sPut *putsvc.Service) *deletesvc.Service { return deletesvc.New( - deletesvc.WithLogger(c.log), - deletesvc.WithHeadService(sGet), - deletesvc.WithSearchService(sSearch), - deletesvc.WithPutService(sPut), - deletesvc.WithNetworkInfo(&delNetInfo{ + sGet, + sSearch, + sPut, + &delNetInfo{ State: c.cfgNetmap.state, tsLifetime: c.cfgObject.tombstoneLifetime, cfg: c, - }), - deletesvc.WithKeyStorage(keyStorage), + }, + keyStorage, + deletesvc.WithLogger(c.log), ) } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 5c09ad123..e208a298b 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -62,16 +62,22 @@ type cfg struct { keyStorage *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(gs *getsvc.Service, + ss *searchsvc.Service, + ps *putsvc.Service, + ni NetworkInfo, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + header: (*headSvcWrapper)(gs), + searcher: (*searchSvcWrapper)(ss), + placer: (*putSvcWrapper)(ps), + netInfo: ni, + keyStorage: ks, + } for i := range opts { opts[i](c) @@ -88,39 +94,3 @@ func WithLogger(l *logger.Logger) Option { c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} } } - -// WithHeadService returns option to set Head service -// to work with object headers. -func WithHeadService(h *getsvc.Service) Option { - return func(c *cfg) { - c.header = (*headSvcWrapper)(h) - } -} - -// WithSearchService returns option to set search service. -func WithSearchService(s *searchsvc.Service) Option { - return func(c *cfg) { - c.searcher = (*searchSvcWrapper)(s) - } -} - -// WithPutService returns option to specify put service. -func WithPutService(p *putsvc.Service) Option { - return func(c *cfg) { - c.placer = (*putSvcWrapper)(p) - } -} - -// WithNetworkInfo returns option to set network information source. -func WithNetworkInfo(netInfo NetworkInfo) Option { - return func(c *cfg) { - c.netInfo = netInfo - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} -- 2.45.2 From 800a685e8477289f8c5ed6f7a58cbc97b823c895 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:21:03 +0300 Subject: [PATCH 200/233] [#294] putsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 16 +++--- pkg/services/object/put/service.go | 84 ++++++++---------------------- 2 files changed, 30 insertions(+), 70 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1f763dd9a..43d68bdcd 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -313,14 +313,14 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } return putsvc.NewService( - putsvc.WithKeyStorage(keyStorage), - putsvc.WithClientConstructor(c.putClientCache), - putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)), - putsvc.WithObjectStorage(os), - putsvc.WithContainerSource(c.cfgObject.cnrSource), - putsvc.WithNetworkMapSource(c.netMapSource), - putsvc.WithNetmapKeys(c), - putsvc.WithNetworkState(c.cfgNetmap.state), + keyStorage, + c.putClientCache, + newCachedMaxObjectSizeSource(c), + os, + c.cfgObject.cnrSource, + c.netMapSource, + c, + c.cfgNetmap.state, putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), putsvc.WithLogger(c.log), ) diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 567a3fea1..7f2600f9c 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -46,8 +46,6 @@ type cfg struct { fmtValidator *object.FormatValidator - fmtValidatorOpts []object.FormatValidatorOption - networkState netmap.State clientConstructor ClientConstructor @@ -55,22 +53,34 @@ type cfg struct { log *logger.Logger } -func defaultCfg() *cfg { - return &cfg{ - remotePool: util.NewPseudoWorkerPool(), - localPool: util.NewPseudoWorkerPool(), - log: &logger.Logger{Logger: zap.L()}, +func NewService(ks *objutil.KeyStorage, + cc ClientConstructor, + ms MaxSizeSource, + os ObjectStorage, + cs container.Source, + ns netmap.Source, + nk netmap.AnnouncedKeys, + nst netmap.State, + opts ...Option) *Service { + c := &cfg{ + remotePool: util.NewPseudoWorkerPool(), + localPool: util.NewPseudoWorkerPool(), + log: &logger.Logger{Logger: zap.L()}, + keyStorage: ks, + clientConstructor: cc, + maxSizeSrc: ms, + localStore: os, + cnrSrc: cs, + netMapSrc: ns, + netmapKeys: nk, + networkState: nst, } -} - -func NewService(opts ...Option) *Service { - c := defaultCfg() for i := range opts { opts[i](c) } - c.fmtValidator = object.NewFormatValidator(c.fmtValidatorOpts...) + c.fmtValidator = object.NewFormatValidator(object.WithLockSource(os), object.WithNetState(nst)) return &Service{ cfg: c, @@ -83,62 +93,12 @@ func (p *Service) Put() (*Streamer, error) { }, nil } -func WithKeyStorage(v *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = v - } -} - -func WithMaxSizeSource(v MaxSizeSource) Option { - return func(c *cfg) { - c.maxSizeSrc = v - } -} - -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.localStore = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithLockSource(v)) - } -} - -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -func WithNetworkMapSource(v netmap.Source) Option { - return func(c *cfg) { - c.netMapSrc = v - } -} - func WithWorkerPools(remote, local util.WorkerPool) Option { return func(c *cfg) { c.remotePool, c.localPool = remote, local } } -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.networkState = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithNetState(v)) - } -} - -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor = v - } -} - func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = l -- 2.45.2 From ec9b73846593666629a3c306369b6ddb0d99a912 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:31:03 +0300 Subject: [PATCH 201/233] [#294] putsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +---- pkg/services/object/put/v2/service.go | 30 +++------------------------ 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 43d68bdcd..37dcc7b3e 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -327,10 +327,7 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2.Service { - return putsvcV2.NewService( - putsvcV2.WithInternalService(sPut), - putsvcV2.WithKeyStorage(keyStorage), - ) + return putsvcV2.NewService(sPut, keyStorage) } func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 78655edc7..db902ae59 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -12,27 +12,15 @@ import ( // Service implements Put operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *putsvc.Service keyStorage *util.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *putsvc.Service, ks *util.KeyStorage) *Service { return &Service{ - cfg: c, + svc: svc, + keyStorage: ks, } } @@ -52,15 +40,3 @@ func (s *Service) Put() (object.PutObjectStream, error) { func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { return s.svc.PutSingle(ctx, req) } - -func WithInternalService(v *putsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} -- 2.45.2 From e8091101c7c2b83d6c1e3a708fc252b3e1f3afe2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:53:37 +0300 Subject: [PATCH 202/233] [#294] searchsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 16 +++--- pkg/services/object/search/service.go | 72 ++++++++------------------- 2 files changed, 27 insertions(+), 61 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 37dcc7b3e..02336bcbf 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -334,16 +334,14 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( - searchsvc.WithLogger(c.log), - searchsvc.WithLocalStorageEngine(ls), - searchsvc.WithClientConstructor(coreConstructor), - searchsvc.WithTraverserGenerator( - traverseGen.WithTraverseOptions( - placement.WithoutSuccessTracking(), - ), + ls, + coreConstructor, + traverseGen.WithTraverseOptions( + placement.WithoutSuccessTracking(), ), - searchsvc.WithNetMapSource(c.netMapSource), - searchsvc.WithKeyStorage(keyStorage), + c.netMapSource, + keyStorage, + searchsvc.WithLogger(c.log), ) } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 708979d79..16b82a620 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -55,17 +55,28 @@ type cfg struct { keyStore *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - clientConstructor: new(clientConstructorWrapper), - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(e *engine.StorageEngine, + cc ClientConstructor, + tg *util.TraverserGenerator, + ns netmap.Source, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + clientConstructor: &clientConstructorWrapper{ + constructor: cc, + }, + localStorage: &storageEngineWrapper{ + storage: e, + }, + traverserGenerator: (*traverseGeneratorWrapper)(tg), + currentEpochReceiver: &nmSrcWrapper{ + nmSrc: ns, + }, + keyStore: ks, + } for i := range opts { opts[i](c) @@ -82,46 +93,3 @@ func WithLogger(l *logger.Logger) Option { c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Search service"))} } } - -// WithLocalStorageEngine returns option to set local storage -// instance. -func WithLocalStorageEngine(e *engine.StorageEngine) Option { - return func(c *cfg) { - c.localStorage = &storageEngineWrapper{ - storage: e, - } - } -} - -// WithClientConstructor returns option to set constructor of remote node clients. -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor.(*clientConstructorWrapper).constructor = v - } -} - -// WithTraverserGenerator returns option to set generator of -// placement traverser to get the objects from containers. -func WithTraverserGenerator(t *util.TraverserGenerator) Option { - return func(c *cfg) { - c.traverserGenerator = (*traverseGeneratorWrapper)(t) - } -} - -// WithNetMapSource returns option to set network -// map storage to receive current network state. -func WithNetMapSource(nmSrc netmap.Source) Option { - return func(c *cfg) { - c.currentEpochReceiver = &nmSrcWrapper{ - nmSrc: nmSrc, - } - } -} - -// WithKeyStorage returns option to set private -// key storage for session tokens and node key. -func WithKeyStorage(store *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStore = store - } -} -- 2.45.2 From c83e7c875f7be3439208758669a81a7298ba28e9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:56:45 +0300 Subject: [PATCH 203/233] [#294] searchsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +--- pkg/services/object/search/v2/service.go | 36 +++--------------------- 2 files changed, 5 insertions(+), 36 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 02336bcbf..1b9f0c817 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -346,10 +346,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav } func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) *searchsvcV2.Service { - return searchsvcV2.NewService( - searchsvcV2.WithInternalService(sSearch), - searchsvcV2.WithKeyStorage(keyStorage), - ) + return searchsvcV2.NewService(sSearch, keyStorage) } func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, diff --git a/pkg/services/object/search/v2/service.go b/pkg/services/object/search/v2/service.go index 17e1bc7e0..78b72ac79 100644 --- a/pkg/services/object/search/v2/service.go +++ b/pkg/services/object/search/v2/service.go @@ -9,28 +9,15 @@ import ( // Service implements Search operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { - svc *searchsvc.Service - + svc *searchsvc.Service keyStorage *objutil.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(s *searchsvc.Service, ks *objutil.KeyStorage) *Service { return &Service{ - cfg: c, + svc: s, + keyStorage: ks, } } @@ -43,18 +30,3 @@ func (s *Service) Search(req *objectV2.SearchRequest, stream objectSvc.SearchStr return s.svc.Search(stream.Context(), *p) } - -// WithInternalService returns option to set entity -// that handles request payload. -func WithInternalService(v *searchsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} -- 2.45.2 From 24eb98889724f58338aeb0874b40de6db1f0cbc5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Jul 2023 12:18:56 +0300 Subject: [PATCH 204/233] [#294] deletesvc: Drop cast Signed-off-by: Dmitrii Stepanov --- pkg/services/object/delete/service.go | 6 +++--- pkg/services/object/delete/util.go | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index e208a298b..b74a4c7ba 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -72,9 +72,9 @@ func New(gs *getsvc.Service, opts ...Option) *Service { c := &cfg{ log: &logger.Logger{Logger: zap.L()}, - header: (*headSvcWrapper)(gs), - searcher: (*searchSvcWrapper)(ss), - placer: (*putSvcWrapper)(ps), + header: &headSvcWrapper{s: gs}, + searcher: &searchSvcWrapper{s: ss}, + placer: &putSvcWrapper{s: ps}, netInfo: ni, keyStorage: ks, } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index b8e8e6324..439abca2b 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -11,11 +11,17 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -type headSvcWrapper getsvc.Service +type headSvcWrapper struct { + s *getsvc.Service +} -type searchSvcWrapper searchsvc.Service +type searchSvcWrapper struct { + s *searchsvc.Service +} -type putSvcWrapper putsvc.Service +type putSvcWrapper struct { + s *putsvc.Service +} type simpleIDWriter struct { ids []oid.ID @@ -30,7 +36,7 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi p.WithRawFlag(true) p.WithAddress(addr) - err := (*getsvc.Service)(w).Head(ctx, p) + err := w.s.Head(ctx, p) if err != nil { return nil, err } @@ -94,7 +100,7 @@ func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]o p.WithContainerID(exec.containerID()) p.WithSearchFilters(fs) - err := (*searchsvc.Service)(w).Search(ctx, p) + err := w.s.Search(ctx, p) if err != nil { return nil, err } @@ -109,7 +115,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { } func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) { - streamer, err := (*putsvc.Service)(w).Put() + streamer, err := w.s.Put() if err != nil { return nil, err } -- 2.45.2 From b8bcfac53106332ce10ec04dfe8e889abd2814a8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Jul 2023 10:21:35 +0300 Subject: [PATCH 205/233] [#510] treesvc: Fix panic in bearer token processing Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 63485a707..b932f6de6 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -110,7 +110,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } tb = *tbCore.Value - if bt.Impersonate() { + if tableFromBearer && bt.Impersonate() { signer = bt.SigningKeyBytes() } } -- 2.45.2 From 8a9fc2c372576e5d8334088fd8620b2126ea1a41 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Jul 2023 10:26:18 +0300 Subject: [PATCH 206/233] [#510] treesvc: Rename `tableFromBearer` to `useBearer` With impersonation, the old name is no longer descriptive. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index b932f6de6..7a466955c 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -84,7 +84,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return nil } - var tableFromBearer bool + var useBearer bool if len(rawBearer) != 0 { if !basicACL.AllowedBearerRules(op) { s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, @@ -92,13 +92,13 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op zap.String("op", op.String()), ) } else { - tableFromBearer = true + useBearer = true } } var tb eacl.Table signer := req.GetSignature().GetKey() - if tableFromBearer && !bt.Impersonate() { + if useBearer && !bt.Impersonate() { if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { return eACLErr(eaclOp, errBearerWrongOwner) } @@ -110,7 +110,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } tb = *tbCore.Value - if tableFromBearer && bt.Impersonate() { + if useBearer && bt.Impersonate() { signer = bt.SigningKeyBytes() } } -- 2.45.2 From 11027945d8de36cfc44c1b5b3980f17be7cd2dc4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 13 Jul 2023 12:13:53 +0300 Subject: [PATCH 207/233] [#479] writecache: Fix writecache fstree flush premature ctx cancel Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/writecache/flush.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 779bac39a..243be4627 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -37,7 +37,6 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { ctx, cancel := context.WithCancel(context.Background()) - defer cancel() ch := c.closeCh c.wg.Add(1) -- 2.45.2 From 397131b0ea3fc742c03bcae97df89dbad67d183f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 16:43:41 +0300 Subject: [PATCH 208/233] [#520] objectcore: Refactor format validator Remove redundant FIXME. Move error to consts. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index abf8f02cb..e6d8174fa 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -64,6 +64,8 @@ var errNoExpirationEpoch = errors.New("missing expiration epoch attribute") var errTombstoneExpiration = errors.New("tombstone body and header contain different expiration values") +var errMissingSignature = errors.New("missing signature") + func defaultCfg() *cfg { return new(cfg) } @@ -135,8 +137,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, u func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { - // TODO(@cthulhu-rider): #468 use "const" error - return errors.New("missing signature") + return errMissingSignature } var sigV2 refs.Signature @@ -157,8 +158,6 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { return v.checkOwnerKey(*obj.OwnerID(), key) } - // FIXME: #1159 perform token verification - return nil } -- 2.45.2 From 486287c2f7802bfa5d6436df566f4fac4942f57e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 14 Jul 2023 16:08:01 +0300 Subject: [PATCH 209/233] [#524] cli: Add impersonate flag for bearer token creation Signed-off-by: Denis Kirillov --- cmd/frostfs-cli/modules/bearer/create.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index 2f1623d9b..a05de9a76 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -24,6 +24,7 @@ const ( ownerFlag = "owner" outFlag = "out" jsonFlag = commonflags.JSON + impersonateFlag = "impersonate" ) var createCmd = &cobra.Command{ @@ -39,15 +40,18 @@ is set to current epoch + n. } func init() { - createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table") + createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") createCmd.Flags().StringP(issuedAtFlag, "i", "", "Epoch to issue token at") createCmd.Flags().StringP(notValidBeforeFlag, "n", "", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") createCmd.Flags().StringP(ownerFlag, "o", "", "Token owner") createCmd.Flags().String(outFlag, "", "File to write token to") createCmd.Flags().Bool(jsonFlag, false, "Output token in JSON") + createCmd.Flags().Bool(impersonateFlag, false, "Mark token as impersonate to consider the token signer as the request owner (mutually exclusive with --eacl flag)") createCmd.Flags().StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + createCmd.MarkFlagsMutuallyExclusive(eaclFlag, impersonateFlag) + _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), issuedAtFlag) @@ -101,6 +105,9 @@ func createToken(cmd *cobra.Command, _ []string) { b.SetIat(iat) b.ForUser(ownerID) + impersonate, _ := cmd.Flags().GetBool(impersonateFlag) + b.SetImpersonate(impersonate) + eaclPath, _ := cmd.Flags().GetString(eaclFlag) if eaclPath != "" { table := eaclSDK.NewTable() -- 2.45.2 From d8e37a827f5361596d6e4262c3864d745de590aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Jul 2023 15:37:35 +0300 Subject: [PATCH 210/233] [#497] config: Add examples and unit tests Add examples and unit tests for tree.authorized_keys section. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/tree/config_test.go | 11 +++++++++++ config/example/node.env | 1 + config/example/node.json | 6 +++++- config/example/node.yaml | 3 +++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config/tree/config_test.go b/cmd/frostfs-node/config/tree/config_test.go index a39aa4553..898f7e715 100644 --- a/cmd/frostfs-node/config/tree/config_test.go +++ b/cmd/frostfs-node/config/tree/config_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -21,10 +22,19 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 0, treeSec.ReplicationChannelCapacity()) require.Equal(t, 0, treeSec.ReplicationWorkerCount()) require.Equal(t, time.Duration(0), treeSec.ReplicationTimeout()) + require.Equal(t, 0, len(treeSec.AuthorizedKeys())) }) const path = "../../../../config/example/node" + var expectedKeys keys.PublicKeys + key, err := keys.NewPublicKeyFromString("0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + key, err = keys.NewPublicKeyFromString("02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + var fileConfigTest = func(c *config.Config) { treeSec := treeconfig.Tree(c) @@ -34,6 +44,7 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 32, treeSec.ReplicationWorkerCount()) require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout()) require.Equal(t, time.Hour, treeSec.SyncInterval()) + require.Equal(t, expectedKeys, treeSec.AuthorizedKeys()) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/config/example/node.env b/config/example/node.env index 143bf0388..089021767 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -36,6 +36,7 @@ FROSTFS_TREE_REPLICATION_CHANNEL_CAPACITY=32 FROSTFS_TREE_REPLICATION_WORKER_COUNT=32 FROSTFS_TREE_REPLICATION_TIMEOUT=5s FROSTFS_TREE_SYNC_INTERVAL=1h +FROSTFS_TREE_AUTHORIZED_KEYS="0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" # gRPC section ## 0 server diff --git a/config/example/node.json b/config/example/node.json index 04aabdd42..e4b85bc81 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -75,7 +75,11 @@ "replication_channel_capacity": 32, "replication_worker_count": 32, "replication_timeout": "5s", - "sync_interval": "1h" + "sync_interval": "1h", + "authorized_keys": [ + "0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0", + "02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" + ] }, "control": { "authorized_keys": [ diff --git a/config/example/node.yaml b/config/example/node.yaml index bc665a688..897f4e15b 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -62,6 +62,9 @@ tree: replication_channel_capacity: 32 replication_timeout: 5s sync_interval: 1h + authorized_keys: # list of hex-encoded public keys that have rights to use the Tree Service with frostfs-cli + - 0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 + - 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56 control: authorized_keys: # list of hex-encoded public keys that have rights to use the Control Service -- 2.45.2 From a0c7045f29a64fec1ef60178cbd3bcbf1100c714 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Jul 2023 11:58:13 +0300 Subject: [PATCH 211/233] [#512] cli: Refactor UX for bearer create command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/bearer/create.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index a05de9a76..b85115047 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -41,8 +41,8 @@ is set to current epoch + n. func init() { createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") - createCmd.Flags().StringP(issuedAtFlag, "i", "", "Epoch to issue token at") - createCmd.Flags().StringP(notValidBeforeFlag, "n", "", "Not valid before epoch") + createCmd.Flags().StringP(issuedAtFlag, "i", "+0", "Epoch to issue token at") + createCmd.Flags().StringP(notValidBeforeFlag, "n", "+0", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") createCmd.Flags().StringP(ownerFlag, "o", "", "Token owner") createCmd.Flags().String(outFlag, "", "File to write token to") @@ -54,8 +54,6 @@ func init() { _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), issuedAtFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), notValidBeforeFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.ExpireAt) _ = cobra.MarkFlagRequired(createCmd.Flags(), ownerFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), outFlag) @@ -72,10 +70,14 @@ func createToken(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't parse --"+notValidBeforeFlag+" flag: %w", err) if iatRelative || expRelative || nvbRelative { + endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + if len(endpoint) == 0 { + commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", fmt.Errorf("'%s' flag value must be specified", commonflags.RPC)) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) currEpoch, err := internalclient.GetCurrentEpoch(ctx, cmd, endpoint) commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", err) -- 2.45.2 From 3e8de14e7d921146f0fdb40307d9cc7fccd8c8e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Jul 2023 16:13:03 +0300 Subject: [PATCH 212/233] [#382] evacuate: Fix unit tests Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/put.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 13ac59f24..b642ea065 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -28,7 +28,7 @@ import ( func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { dir := t.TempDir() - te := testNewEngine(t, WithShardPoolSize(1)). + te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 98c4504e6..2f96b8296 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -154,6 +154,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, putSuccess = true }); err != nil { + e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) close(exitCh) } -- 2.45.2 From 3b66f98f27755e33c00ed845c41eb8996ef55a0e Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 13 Jul 2023 15:04:29 +0300 Subject: [PATCH 213/233] [#519] Use Address.Equals in policer tests Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 42428df23..56dab413e 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -220,11 +220,11 @@ func TestProcessObject(t *testing.T) { WithRemoteObjectHeaderFunc(headFn), WithBuryFunc(buryFn), WithRedundantCopyCallback(func(_ context.Context, a oid.Address) { - require.True(t, eqAddr(a, addr), "unexpected redundant copy callback: a=%v", a) + require.True(t, a.Equals(addr), "unexpected redundant copy callback: a=%v", a) gotRemoveRedundant = true }), WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { - require.True(t, eqAddr(task.Addr, addr), "unexpected replicator task: %+v", task) + require.True(t, task.Addr.Equals(addr), "unexpected replicator task: %+v", task) for _, node := range task.Nodes { gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) } @@ -303,11 +303,6 @@ func TestIteratorContract(t *testing.T) { }, it.calls) } -// TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) -func eqAddr(a, b oid.Address) bool { - return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) -} - type nextResult struct { objs []objectcore.AddressWithType err error -- 2.45.2 From b2487e8cc516c789e4c53e09859450f310689abb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Jul 2023 16:22:38 +0300 Subject: [PATCH 214/233] [#516] node: Do not bootstrap if node is online candidate Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++- cmd/frostfs-node/netmap.go | 19 +++++++++++++------ internal/logs/logs.go | 1 + 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5d7adab29..d22d68ba0 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -337,7 +337,8 @@ type internals struct { apiVersion version.Version healthStatus *atomic.Int32 // is node under maintenance - isMaintenance atomic.Bool + isMaintenance atomic.Bool + isOnlineCandidate bool } // starts node's maintenance. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b7b09a00b..96b866b11 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -227,6 +227,10 @@ func bootstrapNode(c *cfg) { c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } + if c.isOnlineCandidate { + c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap) + return + } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -258,7 +262,8 @@ func initNetmapState(c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) - ni, err := c.netmapInitLocalNodeState(epoch) + var ni *netmapSDK.NodeInfo + ni, c.isOnlineCandidate, err = c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) stateWord := nodeState(ni) @@ -291,27 +296,29 @@ func nodeState(ni *netmapSDK.NodeInfo) string { return "undefined" } -func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, error) { nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() if err != nil { - return nil, err + return nil, false, err } var candidate *netmapSDK.NodeInfo + isOnlineCandidate := false for i := range nmNodes { if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { candidate = &nmNodes[i] + isOnlineCandidate = candidate.IsOnline() break } } node, err := c.netmapLocalNodeState(epoch) if err != nil { - return nil, err + return nil, false, err } if candidate == nil { - return node, nil + return node, false, nil } nmState := nodeState(node) @@ -323,7 +330,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error zap.String("netmap", nmState), zap.String("candidate", candidateState)) } - return candidate, nil + return candidate, isOnlineCandidate, nil } func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 2ba9e6e4b..82e04a16d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -493,4 +493,5 @@ const ( ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" + NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" ) -- 2.45.2 From 8966dd8e35c738306ca7e99accf324e97cb27601 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Jul 2023 17:47:42 +0300 Subject: [PATCH 215/233] [#463] putsvc: Use PutSingle RPC for remote target Signed-off-by: Dmitrii Stepanov --- pkg/core/client/client.go | 1 + pkg/network/cache/multi.go | 9 ++++ pkg/services/object/internal/client/client.go | 48 +++++++++++++++++++ pkg/services/object/put/remote.go | 20 ++++++++ 4 files changed, 78 insertions(+) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 422ca1a1a..8c92901f2 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -13,6 +13,7 @@ import ( type Client interface { ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) + ObjectPutSingle(context.Context, client.PrmObjectPutSingle) (*client.ResObjectPutSingle, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) ObjectGetInit(context.Context, client.PrmObjectGet) (*client.ObjectReader, error) ObjectHead(context.Context, client.PrmObjectHead) (*client.ResObjectHead, error) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 18155849b..98d2f33e7 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -228,6 +228,15 @@ func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutIn return } +func (x *multiClient) ObjectPutSingle(ctx context.Context, p client.PrmObjectPutSingle) (res *client.ResObjectPutSingle, err error) { + err = x.iterateClients(ctx, func(c clientcore.Client) error { + res, err = c.ObjectPutSingle(ctx, p) + return err + }) + + return +} + func (x *multiClient) ContainerAnnounceUsedSpace(ctx context.Context, prm client.PrmAnnounceSpace) (res *client.ResAnnounceSpace, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ContainerAnnounceUsedSpace(ctx, prm) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index cfab77efe..73f4ff7c4 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -449,6 +449,54 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { }, nil } +// PutObjectSingle saves the object in local storage of the remote node with PutSingle RPC. +// +// Client and key must be set. +// +// Returns any error which prevented the operation from completing correctly in error return. +func PutObjectSingle(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObjectSingle") + defer span.End() + + objID, isSet := prm.obj.ID() + if !isSet { + return nil, errors.New("missing object id") + } + + var prmCli client.PrmObjectPutSingle + + prmCli.ExecuteLocal() + + if prm.key != nil { + prmCli.UseKey(prm.key) + } + + if prm.tokenSession != nil { + prmCli.WithinSession(*prm.tokenSession) + } + + if prm.tokenBearer != nil { + prmCli.WithBearerToken(*prm.tokenBearer) + } + + prmCli.WithXHeaders(prm.xHeaders...) + prmCli.SetObject(prm.obj.ToV2()) + + res, err := prm.cli.ObjectPutSingle(ctx, prmCli) + if err != nil { + ReportError(prm.cli, err) + return nil, fmt.Errorf("put single object on client: %w", err) + } + + if err = apistatus.ErrFromStatus(res.Status()); err != nil { + return nil, fmt.Errorf("put single object via client: %w", err) + } + + return &PutObjectRes{ + id: objID, + }, nil +} + // SearchObjectsPrm groups parameters of SearchObjects operation. type SearchObjectsPrm struct { readPrmCommon diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index a5b3f643c..8116243ec 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -13,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type remoteTarget struct { @@ -63,6 +65,15 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetXHeaders(t.commonPrm.XHeaders()) prm.SetObject(t.obj) + res, err := t.putSingle(ctx, prm) + if status.Code(err) != codes.Unimplemented { + return res, err + } + + return t.putStream(ctx, prm) +} + +func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { res, err := internalclient.PutObject(ctx, prm) if err != nil { return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) @@ -71,6 +82,15 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil } +func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { + res, err := internalclient.PutObjectSingle(ctx, prm) + if err != nil { + return nil, fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + } + + return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil +} + // NewRemoteSender creates, initializes and returns new RemoteSender instance. func NewRemoteSender(keyStorage *util.KeyStorage, cons ClientConstructor) *RemoteSender { return &RemoteSender{ -- 2.45.2 From 8b78db74bcfbf0117dedd6e898da4f95f0e191e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 14:58:15 +0300 Subject: [PATCH 216/233] [#463] replicator: Add tracing span Signed-off-by: Dmitrii Stepanov --- pkg/services/replicator/process.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index a54668e12..16bcec9c5 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,7 +6,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -28,6 +31,13 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) ) }() + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleTask", + trace.WithAttributes( + attribute.Stringer("address", task.Addr), + attribute.Int64("number_of_copies", int64(task.NumCopies)), + )) + defer span.End() + if task.Obj == nil { var err error task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) -- 2.45.2 From f0355a453e3e43250e454b5ec544a73c3cf4f965 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 15:08:46 +0300 Subject: [PATCH 217/233] [#463] policer: Remove capacity rebalance logic Current implementation has some quirks. For example, using only half of object.put.pool_size_remote threads tells replicator that is node is 50% loaded, but in reality we could be putting lot's of big objects. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 7 ------- cmd/frostfs-node/object.go | 1 - internal/logs/logs.go | 1 - pkg/services/policer/option.go | 15 --------------- pkg/services/policer/policer_test.go | 7 ------- pkg/services/policer/process.go | 25 ------------------------- 6 files changed, 56 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d22d68ba0..267a05aa8 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -997,13 +997,6 @@ func (c *cfg) needBootstrap() bool { return c.cfgNetmap.needBootstrap } -// ObjectServiceLoad implements system loader interface for policer component. -// It is calculated as size/capacity ratio of "remote object put" worker. -// Returns float value between 0.0 and 1.0. -func (c *cfg) ObjectServiceLoad() float64 { - return float64(c.cfgObject.pool.putRemote.Running()) / float64(c.cfgObject.pool.putRemoteCapacity) -} - type dCmp struct { name string reloadFunc func() error diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1b9f0c817..84411d31b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -259,7 +259,6 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }), policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize), policer.WithPool(c.cfgObject.pool.replication), - policer.WithNodeLoader(c), ) c.workers = append(c.workers, worker{ diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 82e04a16d..a2ff8dcb9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -52,7 +52,6 @@ const ( PolicerRoutineStopped = "routine stopped" // Info in ../node/pkg/services/policer/process.go PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" // Warn in ../node/pkg/services/policer/process.go PolicerPoolSubmission = "pool submission" // Warn in ../node/pkg/services/policer/process.go - PolicerTuneReplicationCapacity = "tune replication capacity" // Debug in ../node/pkg/services/policer/process.go ReplicatorFinishWork = "finish work" // Debug in ../node/pkg/services/replicator/process.go ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 4194353ca..5058b026b 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -41,12 +41,6 @@ type Replicator interface { // RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) -// NodeLoader provides application load statistics. -type nodeLoader interface { - // ObjectServiceLoad returns object service load value in [0:1] range. - ObjectServiceLoad() float64 -} - type cfg struct { headTimeout time.Duration @@ -70,8 +64,6 @@ type cfg struct { taskPool *ants.Pool - loader nodeLoader - maxCapacity int batchSize, cacheSize uint32 @@ -178,10 +170,3 @@ func WithPool(p *ants.Pool) Option { c.taskPool = p } } - -// WithNodeLoader returns option to set FrostFS node load source. -func WithNodeLoader(l nodeLoader) Option { - return func(c *cfg) { - c.loader = l - } -} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 56dab413e..c0aeac515 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -52,7 +52,6 @@ func TestBuryObjectWithoutContainer(t *testing.T) { WithContainerSource(containerSrcFunc(containerSrc)), WithBuryFunc(buryFn), WithPool(pool), - WithNodeLoader(constNodeLoader(0)), ) ctx, cancel := context.WithCancel(context.Background()) @@ -279,7 +278,6 @@ func TestIteratorContract(t *testing.T) { WithContainerSource(containerSrcFunc(containerSrc)), WithBuryFunc(buryFn), WithPool(pool), - WithNodeLoader(constNodeLoader(0)), func(c *cfg) { c.sleepDuration = time.Millisecond }, @@ -372,11 +370,6 @@ type announcedKeysFunc func([]byte) bool func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } -// constNodeLoader is a nodeLoader that always returns a fixed value. -type constNodeLoader float64 - -func (f constNodeLoader) ObjectServiceLoad() float64 { return float64(f) } - // replicatorFunc is a Replicator backed by a function. type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 3b54bf929..1f61c69f4 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -11,7 +11,6 @@ import ( ) func (p *Policer) Run(ctx context.Context) { - go p.poolCapacityWorker(ctx) p.shardPolicyWorker(ctx) p.log.Info(logs.PolicerRoutineStopped) } @@ -65,27 +64,3 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } } } - -func (p *Policer) poolCapacityWorker(ctx context.Context) { - ticker := time.NewTicker(p.rebalanceFreq) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - frostfsSysLoad := p.loader.ObjectServiceLoad() - newCapacity := int((1.0 - frostfsSysLoad) * float64(p.maxCapacity)) - if newCapacity == 0 { - newCapacity++ - } - - if p.taskPool.Cap() != newCapacity { - p.taskPool.Tune(newCapacity) - p.log.Debug(logs.PolicerTuneReplicationCapacity, - zap.Float64("system_load", frostfsSysLoad), - zap.Int("new_capacity", newCapacity)) - } - } - } -} -- 2.45.2 From 4680087711f4fd8a42669a8e8e7eef53b5b8323d Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 17 Jul 2023 13:59:37 +0300 Subject: [PATCH 218/233] [#527] Add support for select-filter expressions in policy playground Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 33 +++++++++++++----- go.mod | 2 +- go.sum | Bin 99007 -> 99007 bytes 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 65255f9c2..97e30a3cd 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -82,15 +82,32 @@ func (repl *policyPlaygroundREPL) handleRemove(args []string) error { } func (repl *policyPlaygroundREPL) handleEval(args []string) error { - policyStr := strings.Join(args, " ") - placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) - if err != nil { - return fmt.Errorf("parsing placement policy: %v", err) - } + policyStr := strings.TrimSpace(strings.Join(args, " ")) + var nodes [][]netmap.NodeInfo nm := repl.netMap() - nodes, err := nm.ContainerNodes(*placementPolicy, nil) - if err != nil { - return fmt.Errorf("building container nodes: %v", err) + + if strings.HasPrefix(policyStr, "CBF") || strings.HasPrefix(policyStr, "SELECT") || strings.HasPrefix(policyStr, "FILTER") { + // Assume that the input is a partial SELECT-FILTER expression. + // Full inline policies always start with UNIQUE or REP keywords, + // or different prefixes when it's the case of an external file. + sfExpr, err := netmap.DecodeSelectFilterString(policyStr) + if err != nil { + return fmt.Errorf("parsing select-filter expression: %v", err) + } + nodes, err = nm.SelectFilterNodes(sfExpr) + if err != nil { + return fmt.Errorf("building select-filter nodes: %v", err) + } + } else { + // Assume that the input is a full policy or input file otherwise. + placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) + if err != nil { + return fmt.Errorf("parsing placement policy: %v", err) + } + nodes, err = nm.ContainerNodes(*placementPolicy, nil) + if err != nil { + return fmt.Errorf("building container nodes: %v", err) + } } for i, ns := range nodes { var ids []string diff --git a/go.mod b/go.mod index e2735b223..7378621c4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index ddbc776336c5b9c02464f6cba41387ea84b1f6d2..2ea573e1e11904d54c28c67a2c07ce1d0b5ffb23 100644 GIT binary patch delta 115 zcmdnr%C^6iZG)wtv$>&xk)?@+Zjz;8nq`Vbvbm+9fkK9%RcLOAPnoy5zPp8YlE1%G zk#?!6xp|tuxwD~{pNp%1p1XmwlWBI4Q;2co29d18{Wp+bhCRY0&uSV^Ueaap;MUr16) zxKn7AL3m+~VYr8hQC_fffVN4NSwOC5dXz=v Date: Mon, 17 Jul 2023 18:10:16 +0300 Subject: [PATCH 219/233] [#511] docs: Remove GitHub mentions from CONTRIBUTING.md Signed-off-by: Dmitrii Stepanov --- CONTRIBUTING.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce617f7f6..53ff7c8df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-node/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-node/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-node/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -27,19 +27,19 @@ Start by forking the `frostfs-node` repository, make changes in a branch and the send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository -Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-node/fork) source +### Set up your Forgejo repository +Fork [FrostFS node upstream](https://git.frostfs.info/TrueCloudLab/frostfs-node) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-node +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-node ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-node -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-node +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-node $ git fetch upstream $ git merge upstream/master ... @@ -58,7 +58,7 @@ $ git checkout -b feature/123-something_awesome After your code changes, make sure - To add test cases for the new code. -- To run `make lint` +- To run `make lint` and `make staticcheck-run` - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. - To run `make test` and `make all` completes. @@ -89,8 +89,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. -- 2.45.2 From a9d04ba86f2edc375ca0c688ebed0658c7728ff3 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 10 Jul 2023 11:03:02 +0300 Subject: [PATCH 220/233] [#244] Remove --local-dump from frostfs-adm config Signed-off-by: Alejandro Lopez --- cmd/frostfs-adm/internal/modules/morph/initialize.go | 6 +++--- cmd/frostfs-adm/internal/modules/morph/local_client.go | 3 +-- cmd/frostfs-adm/internal/modules/morph/root.go | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 9eb867faa..dec1fba20 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -210,11 +210,11 @@ func validateInit(cmd *cobra.Command) error { func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { var c Client var err error - if v.GetString(localDumpFlag) != "" { - if v.GetString(endpointFlag) != "" { + if ldf := cmd.Flags().Lookup(localDumpFlag); ldf != nil && ldf.Changed { + if cmd.Flags().Changed(endpointFlag) { return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", endpointFlag, localDumpFlag) } - c, err = newLocalClient(cmd, v, wallets) + c, err = newLocalClient(cmd, v, wallets, ldf.Value.String()) } else { c, err = getN3Client(v) } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 45d09c387..0367f7479 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -51,7 +51,7 @@ type localClient struct { maxGasInvoke int64 } -func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (*localClient, error) { +func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*localClient, error) { cfg, err := config.LoadFile(v.GetString(protoConfigPath)) if err != nil { return nil, err @@ -87,7 +87,6 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet go bc.Run() - dumpPath := v.GetString(localDumpFlag) if cmd.Name() != "init" { f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0600) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 431be125c..6a9e5b9c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -77,7 +77,6 @@ var ( _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(withdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) _ = viper.BindPFlag(protoConfigPath, cmd.Flags().Lookup(protoConfigPath)) - _ = viper.BindPFlag(localDumpFlag, cmd.Flags().Lookup(localDumpFlag)) }, RunE: initializeSideChainCmd, } -- 2.45.2 From 24dffdac6fa4941b48e38dbf9c6e3d9ba50790d4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 14 Jul 2023 11:11:45 +0300 Subject: [PATCH 221/233] [#521] cli: Add netmap load command to policy playground Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 97e30a3cd..7e9d72a08 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -3,6 +3,7 @@ package container import ( "bufio" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -39,7 +40,7 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { for id, node := range repl.nodes { var attrs []string node.IterateAttributes(func(k, v string) { - attrs = append(attrs, fmt.Sprintf("%s:%s", k, v)) + attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) }) fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ @@ -69,6 +70,40 @@ func (repl *policyPlaygroundREPL) handleAdd(args []string) error { return nil } +func (repl *policyPlaygroundREPL) handleLoad(args []string) error { + if len(args) != 1 { + return fmt.Errorf("too few arguments for command 'add': got %d, want 1", len(args)) + } + + jsonNetmap := map[string]map[string]string{} + + b, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("reading netmap file %q: %v", args[0], err) + } + + if err := json.Unmarshal(b, &jsonNetmap); err != nil { + return fmt.Errorf("decoding json netmap: %v", err) + } + + repl.nodes = make(map[string]netmap.NodeInfo) + for id, attrs := range jsonNetmap { + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + + node := repl.nodes[id] + node.SetPublicKey(key) + for k, v := range attrs { + node.SetAttribute(k, v) + } + repl.nodes[id] = node + } + + return nil +} + func (repl *policyPlaygroundREPL) handleRemove(args []string) error { if len(args) == 0 { return fmt.Errorf("too few arguments for command 'remove': got %d, want >0", len(args)) @@ -150,6 +185,7 @@ func (repl *policyPlaygroundREPL) run() error { cmdHandlers := map[string]func([]string) error{ "ls": repl.handleLs, "add": repl.handleAdd, + "load": repl.handleLoad, "remove": repl.handleRemove, "eval": repl.handleEval, } -- 2.45.2 From 57e7fb5ccfec50de5675ff2a0d9f421428004a7b Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 17 Jul 2023 10:23:51 +0300 Subject: [PATCH 222/233] [#521] cli: Add common aliases to policy playground commands Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 7e9d72a08..1ac41f08c 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -183,10 +183,12 @@ func (repl *policyPlaygroundREPL) run() error { } cmdHandlers := map[string]func([]string) error{ + "list": repl.handleLs, "ls": repl.handleLs, "add": repl.handleAdd, "load": repl.handleLoad, "remove": repl.handleRemove, + "rm": repl.handleRemove, "eval": repl.handleEval, } for reader := bufio.NewReader(os.Stdin); ; { -- 2.45.2 From 94df541426e0433653175b0f9c1f1e97da68ec70 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 19 Jul 2023 16:11:56 +0300 Subject: [PATCH 223/233] [#530] go.mod: Update frostfs-sdk-go and frostfs-api-go versions Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- go.mod | 4 ++-- go.sum | Bin 99007 -> 99007 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7378621c4..2e7d8fd56 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 2ea573e1e11904d54c28c67a2c07ce1d0b5ffb23..c8737ebdc0f41b608ce700934e28be010bf10068 100644 GIT binary patch delta 276 zcmdnr%C^6iZG#Q3i=m~Vfq}8HsjjJoQHrHWvV~!?xrsuCp;bkCYH(FlfQdnHlCN)= zdz!mpn0ZB!vqi4COR!l^PLySKc9y@TkA-$tr+|z!b`K~rH!aTePRepG4-Ks{u*{1{OR=;Fa`Mfu j%5e!zDe=xNNv}wDcAs3&t2ntvh^tvwXuGZu!r;g(X~ delta 275 zcmdnr%C^6iZG#Q3i-C!OrIERbk*<-2si~n^qNPb{QmR6Rp;fSxp@C6Qrfa&kNrq)% za(Sh@rFoTyQ%QMBsc%qHYD!W`c6pFXSa3nH#pDKFCsC{hZNAQ%#O!1aG|AG$LO03M zFwHW>BH7&1&;VpbXl{s4nYX#VyM=d>zrRzFcB!ekd78huv!R!ti>rU0yMeQlX?BoP zi1FllUd_oWf})<-jM7if*UQaM0h#EVrO_8s(i)Y@lxx h=^bwAV&M}KY3$@!05nl?a*YsIv#!u~T_MIv1psPgQ0xEz -- 2.45.2 From b4e72a2dfdede508443e301d12bf619987f428ba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Jul 2023 11:39:17 +0300 Subject: [PATCH 224/233] [#335] treesvc: Sort nodes by Filename in GetSubTree Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 19 +++++-- pkg/local_object_storage/pilorama/forest.go | 12 +++-- .../pilorama/interface.go | 2 +- pkg/local_object_storage/pilorama/types.go | 6 +++ pkg/local_object_storage/shard/tree.go | 2 +- pkg/services/tree/service.go | 49 ++++++++++++++---- pkg/services/tree/service.pb.go | Bin 118702 -> 124265 bytes pkg/services/tree/service.proto | 9 ++++ pkg/services/tree/service_frostfs.pb.go | Bin 57237 -> 58250 bytes pkg/services/tree/service_grpc.pb.go | Bin 18231 -> 19139 bytes pkg/services/tree/types.pb.go | Bin 9911 -> 9911 bytes 12 files changed, 81 insertions(+), 22 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 08c6d26b0..df3e919ec 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s } // TreeGetChildren implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree defer span.End() var err error - var nodes []uint64 + var nodes []pilorama.NodeInfo for _, sh := range e.sortShardsByWeight(cid) { nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) if err != nil { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 53a52433d..a729e2a22 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri } // TreeGetChildren implements the Forest interface. -func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { +func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) { var ( startedAt = time.Now() success = false @@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], nodeID) - var children []uint64 + var result []NodeInfo err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) @@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) c := b.Cursor() for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - children = append(children, binary.LittleEndian.Uint64(k[9:])) + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo := NodeInfo{ + ID: childID, + } + parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) + if found { + childInfo.ParentID = parentID + if err := childInfo.Meta.FromBytes(metaBytes); err != nil { + return err + } + } + result = append(result, childInfo) } return nil }) success = err == nil - return children, metaerr.Wrap(err) + return result, metaerr.Wrap(err) } // TreeList implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 76220c1db..8fb519128 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeGetChildren implements the Forest interface. -func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { +func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str } children := s.tree.getChildren(nodeID) - res := make([]Node, len(children)) - copy(res, children) + res := make([]NodeInfo, 0, len(children)) + for _, childID := range children { + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) + } return res, nil } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 89c752627..ea171a479 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -32,7 +32,7 @@ type Forest interface { TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) + TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/pilorama/types.go b/pkg/local_object_storage/pilorama/types.go index 99918683d..8d8616364 100644 --- a/pkg/local_object_storage/pilorama/types.go +++ b/pkg/local_object_storage/pilorama/types.go @@ -55,3 +55,9 @@ var ( func isAttributeInternal(key string) bool { return key == AttributeFilename } + +type NodeInfo struct { + ID Node + Meta Meta + ParentID Node +} diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 81477325a..7e2c80152 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n } // TreeGetChildren implements the pilorama.Forest interface. -func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 12d970c42..57767f87e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sort" "sync" "sync/atomic" @@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - stack := [][]uint64{{b.GetRootId()}} + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + if err != nil { + return err + } + stack := [][]pilorama.NodeInfo{{{ + ID: b.GetRootId(), + Meta: m, + ParentID: p, + }}} for { if len(stack) == 0 { @@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD continue } - nodeID := stack[len(stack)-1][0] + node := stack[len(stack)-1][0] stack[len(stack)-1] = stack[len(stack)-1][1:] - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID) - if err != nil { - return err - } err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ - NodeId: nodeID, - ParentId: p, - Timestamp: m.Time, - Meta: metaToProto(m.Items), + NodeId: node.ID, + ParentId: node.ParentID, + Timestamp: node.Meta.Time, + Meta: metaToProto(node.Meta.Items), }, }) if err != nil { @@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID) + children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID) + if err != nil { + return err + } + children, err = sortByFilename(children, b.GetOrderBy().GetDirection()) if err != nil { return err } @@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD return nil } +func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) { + switch d { + case GetSubTreeRequest_Body_Order_None: + return nodes, nil + case GetSubTreeRequest_Body_Order_Asc: + if len(nodes) == 0 { + return nodes, nil + } + less := func(i, j int) bool { + return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0 + } + sort.Slice(nodes, less) + return nodes, nil + default: + return nil, fmt.Errorf("unsupported order direction: %s", d.String()) + } +} + // Apply locally applies operation from the remote node to the tree. func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 08664a6d050d1d1618603d7179be6cb5c0331368..63f3e714a2578d648309f46e53a12d3eda9f61f0 100644 GIT binary patch delta 7220 zcmai333QZImOeM3R+11RWC0V3AW)Trkk8{96d zwIWbm-(a&_Ey6hzYYl4M9;g->L-DbrHs#qgU2|-3sj$AnCbXfr^dC@lS{+tvMuu%( zb>n*5eZF;#e%pFqO}(uuP$#z}T)QRRdY`W@>;1Mo39(%HzOufdZcX() z4_Yl2E!UO|pZ&>ccyQtYZpqIry0dqT!qUCb!E~6F3R8mVNVzsmf+?h2o6V`v5KmQO zb%o8|Bm;=Q7S^FcwL8S|)A5PdF4}-yW6RC6Rr~L^OR`5%=CE2)Y7k+Q)l%ngXsoNY z)mU3bSd6|FR5)kaizL5`6&EE{_NMf?{`!i#H8l-^I=dsyR#O+iCx6vCq&;0~-rl(W zZhxJ<$>F#HF5St!7TaOF8fi4pl~wVXQJdxm_MOc&{`%$G$`L+{G?P8rjND~+j5Z+U zk~N(S(P5YwceRl#xGEcWjs^~f`UBAV$`$a@#&r$x+3G~$8McqTOX*-Q%^P+pNSUF=ZzQbrwn4p$IJRZd^yD+ zjwM;du|g>vHm28T$lU3qM5FG7cxvecF<5C9kwq4fH)IWIlrDS7GCVqD?T;p8-0zZN zD?`&I=x3VDR~}z(qgd(Es38=JV|ZUMkl2vjlUWnROLi{U!`{{U zZWfnkkI-85>#3r9j^%2t_xG<7N8b4S>ab4#Jx%l$jM(UmsW3QY+C=`FiE?`|!w%bg zf5R{Z;O_@X0Fx7jmDKzR7B93zBU274K}C~A$$X>O`B<~)HXFp*Hf#T{UTY5%|9!?U zG72Mv_Ex0md;L?9^zKJuRdEcI4dTN04pB1{(LeO(3*q7q=K|vL0)xnS=aR60aH&7? z?61SbM<4rzRvIlX-(e8Fug?4*hSBm_S{ae3SS!opo_Zmg{*CyC17CY{VRG&|MX;2+B0CciJ9_`9|L`mZa( zxFDPi{1#EQ=v!|TmjZg=Yw9CK$#+NP>K9^mjaulx@cnrrr|9^$R+QFS#qv+)i8b{x z;#{MFPlVB4EcufEA}%#3(Exnr8_joDp%$7n_rN>(^g@`qB3nJW__&3p@X_fsmS40` z8rN89Dj!QF<3P$BGKbG}x^Q4wS6`(CX ziVkzRcu^ENxU81UJm@0}f0s`ad0QP7aKQ{pYIFIO0nZ3H1)3>;>IO0uu$HCd|5DQv zbU|u5mAhGPWkAz?SD;rJyBW@3YYj1Dw@2Tg|IAT3x~BRZ3|zr^r3F4(Nt5^=6*MKn zFO}U2#KnaP5dW=?mT`>+3Ys%0Rs}u}flDj@GMb={M@8WM0mFQxlCGZ-r04%}l2

  • 3$ha9BQ=MzPK8SwdrYXEDeRZH)*+zIvl5lfO>{SYHN3_l%>cfzMXZMB@A@ z6vY!(Q8K@6qa1F(g^YY@4w`d&0mXBl13Su7&??8Qz(w0KBrDLnOLX)C z7r9uo!M!(10P!mujA^&g4Q#8TX_nxya85i0K%)XACbggQWHuaHZ=W zxDX0Df&&fkagqh~TXPpyztw0OA1i{Di#LFSh0Dnjp~_t!0##hN0-4qdK}gAJ(8UWu z7s_uwDsRN#PvRZRK|#}E@E0n-c>Un{Y8jRCsYQU81RnOIRu6M7QRIhza6NGRu&;loTd-23H7#t4dy1LqQGDREl` zrAg1LmtfMUB;$CPD61frKU@O-Yobto$h1vrogPOMck8OlrI6|xG)0p_g3qBqgJCT(1I>$c?ro}&5DshW~m_csq1v{u_h#ILjWo5&H!5Z zYRcw56N(-aQ=zG%9Bd-@0Gec@0!lI9D1$fe-3zOA#c1T~*6MK;tc4`YgoNL-7BgZ) z6dqLvrq_bEGiz1L^^8YYkFKRiH6m0Hx2*#UOV*-Vz8;U!v1%S%IlPW$B4%hiI68*B zYFs*fd^runacu%Hz8g>Rk`?E8?3Uw3VLiK^ruNLB*fH{i>Zzvr{9#~(8yl+i&ex^^ zxh)xaEGVVxqbd;+hIa5MzI-rj%(I^`VhalzL{K;RPXE8Ky~NvR>+%& zoEQB2W}2_28!jb~xhmP0g1Sh5fO4cSy#{p3e;H}wz@Y~)lg7xGLVoulN{E<;Pq$Q8 z1=4Iz2lqWhcZRvt?-bTrNWEWA8ueCh*YK?dzE#rL*-qE>>>_Jix_Ws{zv{)yo>Ev@ z$*0>;wvTtB5+7*+RjnkM~Jd{dK*}Nrxo^o)<$un zEYrX(ZLoK=1qnajhLP~#K2-D5QYL-iw04R|j0QHgL(~hA0Y)*B5PUw*?nH(5w&Uh> zZa+pxS3BxgMicJz>bEP-)5kU?Es35#;RzLP+mT{8+ErpuIP9jz$@JY z?d?*g<7MY^;bDo_0lX8^v+NCo?;L839J3+l89%fEftflHz)z%1qM3D;Tu$!TfMZdD z$#=P)tt}S4>a=c3iztwl&+~df9eL!@`d3sHpNC2;+_IG#7%&J7^wdv2TZZ z`q>UD!aO*?gNm`HbQLeC0Hge64M$X+ynOLmZ zNedCg6FaFCPf@$*W;57M&*-3P9CHjvxo(nzCC(Qned|X7gja;ft<_ zYF0HdZV%Zpk+S#D!qAFauZ`7Pak21d{PG^U9ga>Gur9C32*m9Tl|xQwQ&vb~;*Ne0MgalO$z8BHD)3|R7)|5my<>6y{k$w-qzZXt>*uD=n;o*Dt z(E^Jb@3%0FB2qp0_CCt7xPzsUIow$0^x*fQd~PSDjMv)?#}@ifZuSl zN%?uoL1fjwALmToL9~uMY{IP;yiCi*y-W@Z-t{GJrkgt$YHu-34@U*^@%?D(N&6`~ zSaq3v`Vgvm{eE%*JnaCO{Ox{9H|dUKvhOff!w-<%3`7Z-tpgSxpjj4|PG)9~F57hg zHH~kZ4#J7E2WYwl;mSQ;jBM!yympe8&&c(42PxHp3!GXHI@x~^h2qq>{Sdr>k5N44 z5Sq=!FG&Gh9LqmBglhC~^kE1N9wH5J%jM^f4pE93aMD`P!<56M1)I)$#lZ;v-~b}( zJwTE2$Qfs~c=aHbd$rswjXyn1iIW124Kg!p~dzzl*>Ce)gBIO;t zuf9xs`P8GhhTr^0Xa%07w|UKTw1|g#aq=AfBfZGmp2znR{m;>#`NRu!C!c+uhWPZ0 zkbU1rR&momaoqhUJWFom8R1F0ix}|k^lez delta 4686 zcma)AX;4&G7OsPI)6F7_vIr`TJJ3RJG>gdM9u-B6N`eR|Xe272(U1g0BPyB{+tog9 zEGB9+io!T4CX*B)6Kk>u%+${n(QqKoHfFD_lYy3{uZVY z9bb5h$SZDQV$<^?X`n__Y;!#`zjL#b`0%Sj(Q(y7q+YrurkuZXCi05WS$y7|E84C< z5DAyBipCz-GmSUja1!a)^Tp=d(sQziTI3}f(loqQOD>{dhrjsrJAW?FQif=~(Idid zx$(>H6e?~nR^0w@d)ol9?E73%vD97k{^%lpxqC@`aO>)sv;PhmAhz9IEi!&m<0k#! zB2H&(MgBb(8BT-P0ixgst#nZ1cmD9*nTUJW28gRa(F5*_0lvEzs5s8_eb={IoEy%CF-6c>S=k~g_b+<3Jt9gnJ;*Vto2&% z*-u(A_tF$Gps`qtu6hn!Ow-w|jNCaZ4L>qQQzRc-Oo5ylPXXLInnv<3i)jouZ6s&z z+)9I;a42~pS-Gtgd%?>P?|KSl5MKS(@TFyn&FpghBKK#=X3UHNFtdN96vByG$xKT!v*%VJ2KE9efd20~7v*K}ETLzBQ z_`)STrg1z#Sa{!0>kCeI~d`CHpj%KgPOT%~zu4h8*2Kb<-K z`l;Kl5;-pi)$6h#y`*L2-h#|xiCb2H@6x#{)Gb+{2VxE4u0G4pHcH`(si@teK_FtS zKPB=VBlznK0n@v3q=KIUjQkC#XTl;7fAA>;94MEd$&E+tAa5S93VHOq4CNfTOx!M& znnpVI*+?V!<5kdc$7&qwc?MAod5F@VrCAEz+HwNYAntvdVgsevvX$;rBxj43&-N(E zPirBC@@MP7S=H7)YIK%d#W&kwMO_{dp%3>yFV(UD8hCOwdN@PE>P#U$cP#+N4b$+W zUycJ?uBMSM@y)f!Sob7Q=lcOQ1g11ZlOHNmuo5XGxB@#!)BhTC66#TGsI-RTDR1|G88n{4K?!w~; zUNwi@DTmKxAadjeq|qCL7EVndt;W7W;>g=T(`6CNR*B6P(?`EVZ9g^N|GgY{2~(JjVZdW!NyQa!rn1 zUTLx^KULW+!^Ms=IU3S10e*Y|Q5f?91SXWgI_}s;y*H^wm{I+5$l`fz(ZrcF>A$d5C;7>dpFGG1& z8BQoKTsg8`mn(u*=4nVgsGYC_a_J~T=Ks~FD_i0N^Zfn}#4dPR&a^U|`~Cbc84cHt zZJ@OO73a+uzy@BDL*o?Ypb@39YPTcv)p=?!59NPt2g3G3^3^LvgJ`4pP9UnRbamVs zysVL3$fLaU54-U)LA_*ZNjZ9`=edWqAD)M>H_L(g_u^MF%Gimr?v<}&WnMqx8YX+# zT@L7>{PRv4FI(QOG()t~7#IeyL|diEl6R>i0aF+@n=%;eD$mNucm?}Bdl$K4u%r)z z?z~?@k9ZYB_Dg&~ynL&)Y1~y9^x7A{XH&>#y9F=%Vs6!Ln$5I}#aby+6ad;KpS6TJgc&SR=uybn_#qgi2 zP|ODoZ1`2H8!=qXh>}@N;XJjP0)-)2%YW?KG;>o04RO@quFKURE#Xz1Ou` z4aIYD4bGphfe0j0CtDn6H$pAp0t)gL01|-@QZ5t-k|sd#wIg8RExClhpJO)}wK7^v z9DBWnM$zsjZ_!?)<)?SUFKRbT2X|Ar%+A7wW0*JJ?p6U<%=oC$~zkX?D^r3cU+_YY8n zx@?lm#Dg@0Ok94DlHvQ&K}vEm8#%QSa97mhq6sJgHs;o1-@?_kl!8@fEluO9TJRoT zM>ApASVz;z#;x*~nYHya8MAq2JRk+M>$`?WOA$!g;2LzD*Rn}=vNR~;fP5JJ&` z{AC^Bw;v}zH&?w)MF2ISE<-r30SHi}bFa|+zQ`8q!;(c}psjtCW+UiZuOi2eS5QUW zVNiqw4nq@bWLfJEQxXhahiS476GYa^B+KT@Nk_De%x_{BCyqa zgr@4P_{fl_Y}_OIi7q}$0eyh1QL4TC2r6^=D9zEUIY>pIi2_v`QRb4@Xg*?gyhe{9 z^}u6*W#RN=G|dGg4HwwByB?greT>HH`Ycw-VhFn*rxEgx4AXJSaI#sr@;JgOn%(j` zoH6m_ak~^}V*e^yMK$mQh3oK*rCyNTgbNm)ppkl9pqOHL%L$qUHN>63wCg-Up?Z)c zx1!L|Cn-jc?3E=Z%n7tgE^IQO^suU?Kb<6tlf}rhPoiX+MlfgNx|4utYJ%tR zZfGKtPA$6qSCCO2QK~IHDkjbz!}-n7$=qfrEV!9GJmhx=I7zp~#6|tOIq0``n_>_D zy~)7uH{&lQb>FY$ZZqu~E$T11bHd;0EYJUfX7WGU$Wydm*YdDW=skY+OIpZ1Ur`4i z_zIg5XMw`&Jhk&XU(-A`UBE`?d2FodBv)~v*NrnT(pkqvdYsQYgKZ`3$WAyM{{WwF z%4CA=J)YJ@X_)sK`5%au_HJ55{wBO)_4+v*)}-XFU!T1)cYU0}V6b7vT&6c$Qm)WY F`VaT$w>$s< diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index 182d8adb2..ec63d88ec 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -238,6 +238,13 @@ message GetNodeByPathResponse { message GetSubTreeRequest { message Body { + message Order { + enum Direction { + None = 0; + Asc = 1; + } + Direction direction = 1; + } // Container ID in V2 format. bytes container_id = 1; // The name of the tree. @@ -249,6 +256,8 @@ message GetSubTreeRequest { uint32 depth = 4; // Bearer token in V2 format. bytes bearer_token = 5; + // Result ordering. + Order order_by = 6; } // Request body. diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 42b7ba3fc7a175eec8fe18428984e102ad0cc92f..b272e4389907a89b64a5ca73e02db3dce21559f2 100644 GIT binary patch delta 197 zcmbQbpSkNe^M?8|ws`-dl+>chg0k9ju6d=o!I@R58iqOwnRz9~Mj91*E}2EC$t9Wj zd77IOHKs65uCEp3j(5sWsf6llbX&{<(l=S~j-(K#rpbcYb0!y z4N5#_ItmqfU~8Q!;idv*c{e*4nlMiGtZJCdqM$x`N}Mn|R27mj?3-syLV0FMhJtgDONc^AQEI9l z7uSS1M&pU`ax6xMdX^g-7cm+q=jRodC}?nTh5!{njfi(lNr`tW&B^giEy>7F@k`82 zRX_mS$qgbRY(OPST%40XIE#s3Q3+PV161Qw8IV|#0aBT$FD#BlCD2^3;>jNj*mY1{ z;hSHUiqjcjP3#~gAj2QHNC}`B4pt!Go?7CUpOT7j#zu275iEv6+~6z7jphbWsQ6?S zmtfHXakwm+2FHSeoJy>!;XYj_%7&&1=t}<;FDtn z#R#VU+{rZp`z9x;it(fQbaDf?(dI6}?@Y?bhT{l)up8M$rzjw+#TM>Bg}IXz#db`d zWG8~n#>riRB9qrjaBp@Kzs3X#T1*%FZSIt679+m2rC?<3yb}Mx%+{ax8{MdL|p!E@Iql$ZE?uS&&(n#W5vCY4SsXtjXW_mVg8% z-(und^ECu!0Qpit_Aih)t8ac;Dnw<9&=#=D4a|ICm8*mgg8BcLxi?P`(PRd3I75n3 zQ++avOCVa@B&L8w*&GWBaw?&wNbUwJbWbhuFYw7vSDHLQ#BK5~Y3|LFrLHl7qLW+ni!|608>HY>@aZd+yC=93Fx_iRwU* JVeYOqtN^?_ehUBq diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index 45d889177d01f3b6c8ed8b3959d5e7dcd24c4ed9..b4d6981ef92d8434e0d3ffa6370d8cb860db9ce4 100644 GIT binary patch delta 15 Wcmdn)yWMxf5*8yvJ Date: Thu, 13 Jul 2023 11:50:13 +0300 Subject: [PATCH 225/233] [#335] treesvc: Fix inmemory unit tests and nil meta items Bolt forest saves empty slice of items. Now inmemory forest does it the same way. Signed-off-by: Dmitrii Stepanov --- .../pilorama/forest_test.go | 19 +++++++++++++------ pkg/local_object_storage/pilorama/inmemory.go | 8 ++++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 9e8e98863..5c143d3eb 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -147,16 +147,23 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { treeAdd(t, 2, 0) treeAdd(t, 7, 0) - testGetChildren := func(t *testing.T, nodeID Node, expected []Node) { + testGetChildren := func(t *testing.T, nodeID Node, expected []NodeInfo) { actual, err := s.TreeGetChildren(context.Background(), cid, treeID, nodeID) require.NoError(t, err) require.ElementsMatch(t, expected, actual) } - testGetChildren(t, 0, []uint64{10, 2, 7}) - testGetChildren(t, 10, []uint64{3, 6}) + testGetChildren(t, 0, []NodeInfo{ + {ID: 10, Meta: Meta{Time: 1, Items: []KeyValue{}}}, + {ID: 2, Meta: Meta{Time: 5, Items: []KeyValue{}}}, + {ID: 7, Meta: Meta{Time: 6, Items: []KeyValue{}}}, + }) + testGetChildren(t, 10, []NodeInfo{ + {ID: 3, ParentID: 10, Meta: Meta{Time: 2, Items: []KeyValue{}}}, + {ID: 6, ParentID: 10, Meta: Meta{Time: 3, Items: []KeyValue{}}}, + }) testGetChildren(t, 3, nil) - testGetChildren(t, 6, []uint64{11}) + testGetChildren(t, 6, []NodeInfo{{ID: 11, ParentID: 6, Meta: Meta{Time: 4, Items: []KeyValue{}}}}) testGetChildren(t, 11, nil) testGetChildren(t, 2, nil) testGetChildren(t, 7, nil) @@ -495,11 +502,11 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ nodes, err := s.TreeGetChildren(ctx, cid, treeID, RootID) require.NoError(t, err) - require.Equal(t, []Node{1}, nodes) + require.Equal(t, []NodeInfo{{ID: 1, ParentID: RootID, Meta: meta[0]}}, nodes) nodes, err = s.TreeGetChildren(ctx, cid, treeID, 1) require.NoError(t, err) - require.Equal(t, []Node{2}, nodes) + require.Equal(t, []NodeInfo{{ID: 2, ParentID: 1, Meta: meta[1]}}, nodes) } t.Run("expected", func(t *testing.T) { diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index 1bde312ac..c9f5df3b7 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -68,10 +68,14 @@ func (s *memoryTree) Apply(op *Move) error { // do performs a single move operation on a tree. func (s *memoryTree) do(op *Move) move { + m := op.Meta + if m.Items == nil { + m.Items = []KeyValue{} + } lm := move{ Move: Move{ Parent: op.Parent, - Meta: op.Meta, + Meta: m, Child: op.Child, }, } @@ -91,7 +95,7 @@ func (s *memoryTree) do(op *Move) move { p.Meta.Time = op.Time } - p.Meta = op.Meta + p.Meta = m p.Parent = op.Parent s.tree.infoMap[op.Child] = p -- 2.45.2 From eed594431f9260e9dbf65a9e86c1be33b2f645ca Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 11:51:19 +0300 Subject: [PATCH 226/233] [#335] treesvc: Add GetSubTree ordering unit test Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/getsubtree_test.go | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index dc4ce29aa..88a5b5e06 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -3,6 +3,8 @@ package tree import ( "context" "errors" + "path" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -119,6 +121,65 @@ func TestGetSubTree(t *testing.T) { }) } +func TestGetSubTreeOrderAsc(t *testing.T) { + d := pilorama.CIDDescriptor{CID: cidtest.ID(), Size: 1} + treeID := "sometree" + p := pilorama.NewMemoryForest() + + tree := []struct { + path []string + id uint64 + }{ + {path: []string{"dir1"}}, + {path: []string{"dir2"}}, + {path: []string{"dir1", "sub1"}}, + {path: []string{"dir2", "sub1"}}, + {path: []string{"dir2", "sub2"}}, + {path: []string{"dir2", "sub1", "subsub1"}}, + } + + for i := range tree { + path := tree[i].path + meta := []pilorama.KeyValue{ + {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} + + lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) + require.NoError(t, err) + require.Equal(t, 1, len(lm)) + tree[i].id = lm[0].Child + } + + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + // GetSubTree must return child only after is has returned the parent. + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + + paths := make([]string, 0, len(acc.seen)) + for i := range acc.seen { + if i == 0 { + continue + } + found := false + for j := range tree { + if acc.seen[i].Body.NodeId == tree[j].id { + found = true + paths = append(paths, path.Join(tree[j].path...)) + } + } + require.True(t, found, "unknown node") + } + + require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { + return paths[i] < paths[j] + })) +} + var ( errSubTreeSend = errors.New("send finished with error") errSubTreeSendAfterError = errors.New("send was invoked after an error occurred") -- 2.45.2 From 5a4054eeb67436147939af4617aa0215b93fb955 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Jul 2023 11:23:26 +0300 Subject: [PATCH 227/233] [#535] .forgejo: Add vulncheck workflow Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/vulncheck.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .forgejo/workflows/vulncheck.yml diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml new file mode 100644 index 000000000..9390ad2d6 --- /dev/null +++ b/.forgejo/workflows/vulncheck.yml @@ -0,0 +1,22 @@ +name: Vulncheck +on: [pull_request] + +jobs: + vulncheck: + name: Vulncheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Run govulncheck + run: govulncheck ./... -- 2.45.2 From fef172c5b03f424953601eaa281b70874a4bd087 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:04:24 +0300 Subject: [PATCH 228/233] [#6] services/util: Simplify `response.Service` It has only 1 parameter, which is obligatory. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 2 +- pkg/services/util/response/service.go | 40 ++++----------------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 267a05aa8..c088d2f07 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -622,7 +622,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt key: key, binPublicKey: key.PublicKey().Bytes(), localAddr: netAddr, - respSvc: response.NewService(response.WithNetworkState(netState)), + respSvc: response.NewService(netState), clientCache: cache.NewSDKClientCache(cacheOpts), bgClientCache: cache.NewSDKClientCache(cacheOpts), putClientCache: cache.NewSDKClientCache(cacheOpts), diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index a63d5343b..005a643e5 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -11,45 +11,24 @@ import ( // Service represents universal v2 service // that sets response meta header values. type Service struct { - cfg *cfg -} - -// Option is an option of Service constructor. -type Option func(*cfg) - -type cfg struct { version refs.Version state netmap.State } -func defaultCfg() *cfg { - var c cfg - - version.Current().WriteToV2(&c.version) - - return &c -} - // NewService creates, initializes and returns Service instance. -func NewService(opts ...Option) *Service { - c := defaultCfg() - - for i := range opts { - opts[i](c) - } - - return &Service{ - cfg: c, - } +func NewService(nmState netmap.State) *Service { + s := &Service{state: nmState} + version.Current().WriteToV2(&s.version) + return s } // SetMeta sets adds meta-header to resp. func (s *Service) SetMeta(resp util.ResponseMessage) { meta := new(session.ResponseMetaHeader) - meta.SetVersion(&s.cfg.version) + meta.SetVersion(&s.version) meta.SetTTL(1) // FIXME: #1160 TTL must be calculated - meta.SetEpoch(s.cfg.state.CurrentEpoch()) + meta.SetEpoch(s.state.CurrentEpoch()) if origin := resp.GetMetaHeader(); origin != nil { // FIXME: #1160 what if origin is set by local server? @@ -58,10 +37,3 @@ func (s *Service) SetMeta(resp util.ResponseMessage) { resp.SetMetaHeader(meta) } - -// WithNetworkState returns option to set network state of Service. -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.state = v - } -} -- 2.45.2 From 372160d04807b12ffddacaa8ed949b93b29f98be Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:26:43 +0300 Subject: [PATCH 229/233] [#6] services/util: Remove `SignService.HandleUnaryRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/accounting/sign.go | 15 +--- pkg/services/container/sign.go | 119 ++++++++++---------------------- pkg/services/netmap/sign.go | 51 ++++---------- pkg/services/object/sign.go | 68 ++++++------------ pkg/services/session/sign.go | 17 ++--- pkg/services/util/sign.go | 45 ++++++------ 6 files changed, 97 insertions(+), 218 deletions(-) diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index e98d9b3af..9efb063f5 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -22,17 +22,6 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - func() util.ResponseMessage { - return new(accounting.BalanceResponse) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 9e77e2e21..55125335f 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -22,113 +22,64 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - func() util.ResponseMessage { - return new(container.PutResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.PutResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.PutResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(container.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.DeleteResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - func() util.ResponseMessage { - return new(container.GetResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.GetResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - func() util.ResponseMessage { - return new(container.ListResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.ListResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.ListResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.SetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.SetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.SetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.GetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.GetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - func() util.ResponseMessage { - return new(container.AnnounceUsedSpaceResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.AnnounceUsedSpaceResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.AnnounceUsedSpaceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 85b19d862..e665519e1 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -24,49 +24,28 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) LocalNodeInfo( ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.LocalNodeInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.LocalNodeInfoResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.LocalNodeInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.NetworkInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.NetworkInfoResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.NetworkInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - func() util.ResponseMessage { - return new(netmap.SnapshotResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.SnapshotResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.SnapshotResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 5b3578e29..f160b4c56 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -105,35 +105,21 @@ func (s *SignService) Put() (PutObjectStream, error) { } func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - func() util.ResponseMessage { - return new(object.HeadResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.HeadResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.HeadResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.PutSingle(ctx, req.(*object.PutSingleRequest)) - }, - func() util.ResponseMessage { - return new(object.PutSingleResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.PutSingleResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.PutSingleResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { @@ -172,19 +158,12 @@ func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) err } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(object.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.DeleteResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { @@ -209,17 +188,10 @@ func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRang } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - func() util.ResponseMessage { - return new(object.GetRangeHashResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeHashResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.GetRangeHashResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 1156dc538..33c0a531c 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -22,17 +22,10 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - func() util.ResponseMessage { - return new(session.CreateResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(session.CreateResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*session.CreateResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index cb4be3084..e41575663 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -172,44 +172,39 @@ func (s *SignService) HandleServerStreamRequest( return nil } -func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler UnaryHandler, blankResp ResponseConstructor) (ResponseMessage, error) { +func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err error) error { // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var ( - resp ResponseMessage - err error - ) - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - var sigErr apistatus.SignatureVerification - sigErr.SetMessage(err.Error()) - - err = sigErr - } else { - // process request - resp, err = handler(ctx, req) - } + statusSupported := isStatusSupported(req) if err != nil { if !statusSupported { - return nil, err + return err } - resp = blankResp() - setStatusV2(resp, err) } // sign the response - if err = signResponse(s.key, resp, statusSupported); err != nil { - return nil, err - } + return signResponse(s.key, resp, statusSupported) +} - return resp, nil +func (s *SignService) VerifyRequest(req RequestMessage) error { + if err := signature.VerifyServiceMessage(req); err != nil { + var sigErr apistatus.SignatureVerification + sigErr.SetMessage(err.Error()) + return sigErr + } + return nil +} + +// EnsureNonNilResponse creates an appropriate response struct if it is nil. +func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { + if resp != nil { + return resp, err + } + return new(T), err } func isStatusSupported(req RequestMessage) bool { -- 2.45.2 From c2617baf6323bde3fa387aa1741c277715d7eb97 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 20:01:13 +0300 Subject: [PATCH 230/233] [#6] services/util: Remove remaining stream wrappers Signed-off-by: Evgenii Stratonikov --- pkg/services/accounting/sign.go | 2 +- pkg/services/container/sign.go | 28 ++--- pkg/services/netmap/sign.go | 12 +- pkg/services/object/sign.go | 188 ++++++++++++++++---------------- pkg/services/session/sign.go | 4 +- pkg/services/util/sign.go | 188 ++++---------------------------- 6 files changed, 139 insertions(+), 283 deletions(-) diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index 9efb063f5..be7b08a39 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -23,5 +23,5 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 55125335f..b336f19c3 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -24,62 +24,62 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.PutResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.DeleteResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.ListResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.SetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.AnnounceUsedSpaceResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index e665519e1..2d01164a3 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -26,26 +26,26 @@ func (s *signService) LocalNodeInfo( req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.LocalNodeInfoResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.NetworkInfoResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.SnapshotResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index f160b4c56..c516872e3 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -18,27 +18,30 @@ type SignService struct { } type searchStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + SearchStream + statusSupported bool + sigSvc *util.SignService nonEmptyResp bool // set on first Send call } type getStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectStream + statusSupported bool + sigSvc *util.SignService } type putStreamSigner struct { - stream *util.RequestMessageStreamer + sigSvc *util.SignService + stream PutObjectStream + statusSupported bool + err error } type getRangeStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectRangeStream + statusSupported bool + sigSvc *util.SignService } func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { @@ -50,37 +53,50 @@ func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { } func (s *getStreamSigner) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectStream.Send(resp) } func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }, - func() util.ResponseMessage { - return new(object.GetResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.Get(req, &getStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.Get(req, &getStreamSigner{ + GetObjectStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + s.statusSupported = util.IsStatusSupported(req) + + if s.err = s.sigSvc.VerifyRequest(req); s.err != nil { + return util.ErrAbortStream + } + if s.err = s.stream.Send(ctx, req); s.err != nil { + return util.ErrAbortStream + } + return nil } -func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv(ctx) - if err != nil { - return nil, fmt.Errorf("could not receive response: %w", err) +func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutResponse, err error) { + if s.err != nil { + err = s.err + resp = new(object.PutResponse) + } else { + resp, err = s.stream.CloseAndRecv(ctx) + if err != nil { + return nil, fmt.Errorf("could not close stream and receive response: %w", err) + } } - return r.(*object.PutResponse), nil + return resp, s.sigSvc.SignResponse(s.statusSupported, resp, err) } func (s *SignService) Put() (PutObjectStream, error) { @@ -90,108 +106,96 @@ func (s *SignService) Put() (PutObjectStream, error) { } return &putStreamSigner{ - stream: s.sigSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - func() util.ResponseMessage { - return new(object.PutResponse) - }, - ), + stream: stream, + sigSvc: s.sigSvc, }, nil } func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.HeadResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.PutSingleResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.SearchStream.Send(resp) } func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }, - func() util.ResponseMessage { - return new(object.SearchResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - stream := &searchStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - } + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.SearchResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } - err := s.svc.Search(req, stream) - - if err == nil && !stream.nonEmptyResp { - // The higher component does not write any response in the case of an empty result (which is correct). - // With the introduction of status returns at least one answer must be signed and sent to the client. - // This approach is supported by clients who do not know how to work with statuses (one could make - // a switch according to the protocol version from the request, but the costs of sending an empty - // answer can be neglected due to the gradual refusal to use the "old" clients). - return stream.Send(new(object.SearchResponse)) - } - - return err - }, - ) + ss := &searchStreamSigner{ + SearchStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + } + err := s.svc.Search(req, ss) + if err == nil && !ss.nonEmptyResp { + // The higher component does not write any response in the case of an empty result (which is correct). + // With the introduction of status returns at least one answer must be signed and sent to the client. + // This approach is supported by clients who do not know how to work with statuses (one could make + // a switch according to the protocol version from the request, but the costs of sending an empty + // answer can be neglected due to the gradual refusal to use the "old" clients). + return stream.Send(new(object.SearchResponse)) + } + return err } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.DeleteResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectRangeStream.Send(resp) } func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }, - func() util.ResponseMessage { - return new(object.GetRangeResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.GetRange(req, &getRangeStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.GetRange(req, &getRangeStreamSigner{ + GetObjectRangeStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.GetRangeHashResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 33c0a531c..ffce0621e 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -24,8 +24,8 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(session.CreateResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index e41575663..a3e0c946e 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -1,7 +1,6 @@ package util import ( - "context" "crypto/ecdsa" "errors" "fmt" @@ -21,163 +20,23 @@ type ResponseMessage interface { SetMetaHeader(*session.ResponseMetaHeader) } -type UnaryHandler func(context.Context, any) (ResponseMessage, error) - type SignService struct { key *ecdsa.PrivateKey } -type ResponseMessageWriter func(ResponseMessage) error - -type ServerStreamHandler func(context.Context, any) (ResponseMessageReader, error) - -type ResponseMessageReader func() (ResponseMessage, error) - var ErrAbortStream = errors.New("abort message stream") -type ResponseConstructor func() ResponseMessage - -type RequestMessageWriter func(context.Context, any) error - -type ClientStreamCloser func(context.Context) (ResponseMessage, error) - -type RequestMessageStreamer struct { - key *ecdsa.PrivateKey - - send RequestMessageWriter - - close ClientStreamCloser - - respCons ResponseConstructor - - statusSupported bool - - sendErr error -} - func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { return &SignService{ key: key, } } -func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { - // req argument should be strengthen with type RequestMessage - s.statusSupported = isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = s.send(ctx, req) - } - - if err != nil { - if !s.statusSupported { - return err - } - - s.sendErr = err - - return ErrAbortStream - } - - return nil -} - -func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMessage, error) { - var ( - resp ResponseMessage - err error - ) - - if s.sendErr != nil { - err = s.sendErr - } else { - resp, err = s.close(ctx) - if err != nil { - err = fmt.Errorf("could not close stream and receive response: %w", err) - } - } - - if err != nil { - if !s.statusSupported { - return nil, err - } - - resp = s.respCons() - - setStatusV2(resp, err) - } - - if err = signResponse(s.key, resp, s.statusSupported); err != nil { - return nil, err - } - - return resp, nil -} - -func (s *SignService) CreateRequestStreamer(sender RequestMessageWriter, closer ClientStreamCloser, blankResp ResponseConstructor) *RequestMessageStreamer { - return &RequestMessageStreamer{ - key: s.key, - send: sender, - close: closer, - - respCons: blankResp, - } -} - -func (s *SignService) HandleServerStreamRequest( - req any, - respWriter ResponseMessageWriter, - blankResp ResponseConstructor, - respWriterCaller func(ResponseMessageWriter) error, -) error { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = respWriterCaller(func(resp ResponseMessage) error { - if err := signResponse(s.key, resp, statusSupported); err != nil { - return err - } - - return respWriter(resp) - }) - } - - if err != nil { - if !statusSupported { - return err - } - - resp := blankResp() - - setStatusV2(resp, err) - - _ = signResponse(s.key, resp, false) // panics or returns nil with false arg - - return respWriter(resp) - } - - return nil -} - -func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err error) error { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req) - +// SignResponse response with private key via signature.SignServiceMessage. +// The signature error affects the result depending on the protocol version: +// - if status return is supported, panics since we cannot return the failed status, because it will not be signed. +// - otherwise, returns error in order to transport it directly. +func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, err error) error { if err != nil { if !statusSupported { return err @@ -186,8 +45,18 @@ func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err setStatusV2(resp, err) } - // sign the response - return signResponse(s.key, resp, statusSupported) + err = signature.SignServiceMessage(s.key, resp) + if err != nil { + err = fmt.Errorf("could not sign response: %w", err) + + if statusSupported { + // We can't pass this error as status code since response will be unsigned. + // Isn't expected in practice, so panic is ok here. + panic(err) + } + } + + return err } func (s *SignService) VerifyRequest(req RequestMessage) error { @@ -207,7 +76,9 @@ func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { return new(T), err } -func isStatusSupported(req RequestMessage) bool { +// IsStatusSupported returns true iff request version implies expecting status return. +// This allows us to handle protocol versions <=2.10 (API statuses was introduced in 2.11 only). +func IsStatusSupported(req RequestMessage) bool { version := req.GetMetaHeader().GetVersion() mjr := version.GetMajor() @@ -223,22 +94,3 @@ func setStatusV2(resp ResponseMessage, err error) { session.SetStatus(resp, apistatus.ToStatusV2(apistatus.ErrToStatus(err))) } - -// signs response with private key via signature.SignServiceMessage. -// The signature error affects the result depending on the protocol version: -// - if status return is supported, panics since we cannot return the failed status, because it will not be signed; -// - otherwise, returns error in order to transport it directly. -func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { - err := signature.SignServiceMessage(key, resp) - if err != nil { - err = fmt.Errorf("could not sign response: %w", err) - - if statusSupported { - // We can't pass this error as status code since response will be unsigned. - // Isn't expected in practice, so panic is ok here. - panic(err) - } - } - - return err -} -- 2.45.2 From 448b48287c7b4df99cbe24426747f2b28bfa68df Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 31 Dec 2022 09:43:13 +0300 Subject: [PATCH 231/233] [#6] services/util: Do not panic in sign function Signed-off-by: Evgenii Stratonikov --- pkg/services/util/sign.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index a3e0c946e..a26bd311c 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -47,16 +47,10 @@ func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, e err = signature.SignServiceMessage(s.key, resp) if err != nil { - err = fmt.Errorf("could not sign response: %w", err) - - if statusSupported { - // We can't pass this error as status code since response will be unsigned. - // Isn't expected in practice, so panic is ok here. - panic(err) - } + return fmt.Errorf("could not sign response: %w", err) } - return err + return nil } func (s *SignService) VerifyRequest(req RequestMessage) error { -- 2.45.2 From 5ff82ff04f673f728ee7b9a081d9a79809681c2c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 10 Jan 2023 15:10:54 +0300 Subject: [PATCH 232/233] [#6] services/object: Simplify local/remote targets We do not use the return result from Close() and we always execute both methods in succession. It makes sense to unite them. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 8 ++--- pkg/services/object/put/local.go | 34 ++++++--------------- pkg/services/object/put/remote.go | 41 +++++++++----------------- pkg/services/object/put/streamer.go | 2 +- 4 files changed, 27 insertions(+), 58 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 408ee099c..cf5cc558e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -18,8 +18,7 @@ import ( ) type preparedObjectTarget interface { - WriteObject(*objectSDK.Object, object.ContentMeta) error - Close(ctx context.Context) (*transformer.AccessIdentifiers, error) + WriteObject(context.Context, *objectSDK.Object, object.ContentMeta) error } type distributedTarget struct { @@ -170,10 +169,9 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error target := t.nodeTargetInitializer(node) - if err := target.WriteObject(t.obj, t.objMeta); err != nil { + err := target.WriteObject(ctx, t.obj, t.objMeta) + if err != nil { return fmt.Errorf("could not write header: %w", err) - } else if _, err := target.Close(ctx); err != nil { - return fmt.Errorf("could not close object stream: %w", err) } return nil } diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index be202892a..54649adc7 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -7,7 +7,6 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) // ObjectStorage is an object storage interface. @@ -27,41 +26,26 @@ type ObjectStorage interface { type localTarget struct { storage ObjectStorage - - obj *objectSDK.Object - meta objectCore.ContentMeta } -func (t *localTarget) WriteObject(obj *objectSDK.Object, meta objectCore.ContentMeta) error { - t.obj = obj - t.meta = meta - - return nil -} - -func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - switch t.meta.Type() { +func (t localTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { + switch meta.Type() { case objectSDK.TypeTombstone: - err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) + return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } case objectSDK.TypeLock: - err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Lock(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) + return fmt.Errorf("could not lock object from lock objects locally: %w", err) } default: // objects that do not change meta storage } - if err := t.storage.Put(ctx, t.obj); err != nil { //TODO - return nil, fmt.Errorf("(%T) could not put object to local storage: %w", t, err) + if err := t.storage.Put(ctx, obj); err != nil { + return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } - - id, _ := t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index 8116243ec..ee8d64e7a 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -24,8 +23,6 @@ type remoteTarget struct { nodeInfo clientcore.NodeInfo - obj *objectSDK.Object - clientConstructor ClientConstructor } @@ -44,16 +41,10 @@ type RemotePutPrm struct { obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(obj *objectSDK.Object, _ objectcore.ContentMeta) error { - t.obj = obj - - return nil -} - -func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *remoteTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, _ objectcore.ContentMeta) error { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { - return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) + return fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) } var prm internalclient.PutObjectPrm @@ -63,32 +54,30 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetSessionToken(t.commonPrm.SessionToken()) prm.SetBearerToken(t.commonPrm.BearerToken()) prm.SetXHeaders(t.commonPrm.XHeaders()) - prm.SetObject(t.obj) + prm.SetObject(obj) - res, err := t.putSingle(ctx, prm) + err = t.putSingle(ctx, prm) if status.Code(err) != codes.Unimplemented { - return res, err + return err } return t.putStream(ctx, prm) } -func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { - res, err := internalclient.PutObject(ctx, prm) +func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObject(ctx, prm) if err != nil { - return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + return fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } - - return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil + return nil } -func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { - res, err := internalclient.PutObjectSingle(ctx, prm) +func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObjectSingle(ctx, prm) if err != nil { - return nil, fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + return fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } - - return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil + return nil } // NewRemoteSender creates, initializes and returns new RemoteSender instance. @@ -134,9 +123,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { return fmt.Errorf("parse client node info: %w", err) } - if err := t.WriteObject(p.obj, objectcore.ContentMeta{}); err != nil { - return fmt.Errorf("(%T) could not send object header: %w", s, err) - } else if _, err := t.Close(ctx); err != nil { + if err := t.WriteObject(ctx, p.obj, objectcore.ContentMeta{}); err != nil { return fmt.Errorf("(%T) could not send object: %w", s, err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 2dccafa47..80b1c2541 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -224,7 +224,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { getWorkerPool: p.getWorkerPool, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { - return &localTarget{ + return localTarget{ storage: p.localStore, } } -- 2.45.2 From 32c77f3a2376acad9affbd901f8e451be4348c16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Jul 2023 10:07:38 +0300 Subject: [PATCH 233/233] [#537] node: Add runtime.memory_limit config parameter This parameter allows to set soft memory limit for Go GC. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 4 +++ cmd/frostfs-node/config/runtime/config.go | 23 ++++++++++++++ .../config/runtime/config_test.go | 30 +++++++++++++++++++ cmd/frostfs-node/main.go | 1 + cmd/frostfs-node/runtime.go | 26 ++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 3 ++ config/example/node.yaml | 3 ++ docs/storage-node-configuration.md | 13 ++++++++ internal/logs/logs.go | 2 ++ 10 files changed, 107 insertions(+) create mode 100644 cmd/frostfs-node/config/runtime/config.go create mode 100644 cmd/frostfs-node/config/runtime/config_test.go create mode 100644 cmd/frostfs-node/runtime.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c088d2f07..283cf501a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1055,6 +1055,10 @@ func (c *cfg) reloadConfig(ctx context.Context) { } components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"runtime", func() error { + setRuntimeParameters(c) + return nil + }}) components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) if updated { diff --git a/cmd/frostfs-node/config/runtime/config.go b/cmd/frostfs-node/config/runtime/config.go new file mode 100644 index 000000000..ad6cce43b --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config.go @@ -0,0 +1,23 @@ +package runtime + +import ( + "math" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" +) + +const ( + subsection = "runtime" + memoryLimitDefault = math.MaxInt64 +) + +// GCMemoryLimitBytes returns the value of "soft_memory_limit" config parameter from "runtime" section. +func GCMemoryLimitBytes(c *config.Config) int64 { + l := config.SizeInBytesSafe(c.Sub(subsection), "soft_memory_limit") + + if l > 0 { + return int64(l) + } + + return memoryLimitDefault +} diff --git a/cmd/frostfs-node/config/runtime/config_test.go b/cmd/frostfs-node/config/runtime/config_test.go new file mode 100644 index 000000000..1bfa42ad8 --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config_test.go @@ -0,0 +1,30 @@ +package runtime + +import ( + "math" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestGCMemoryLimit(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Equal(t, int64(math.MaxInt64), GCMemoryLimitBytes(empty)) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + require.Equal(t, int64(1073741824), GCMemoryLimitBytes(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 425cf25a0..bf872da03 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -84,6 +84,7 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() + setRuntimeParameters(c) metrics, _ := metricsComponent(c) initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) diff --git a/cmd/frostfs-node/runtime.go b/cmd/frostfs-node/runtime.go new file mode 100644 index 000000000..d858ba490 --- /dev/null +++ b/cmd/frostfs-node/runtime.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "runtime/debug" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/runtime" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "go.uber.org/zap" +) + +func setRuntimeParameters(c *cfg) { + if len(os.Getenv("GOMEMLIMIT")) != 0 { + // default limit < yaml limit < app env limit < GOMEMLIMIT + c.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + return + } + + memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg) + previous := debug.SetMemoryLimit(memLimitBytes) + if memLimitBytes != previous { + c.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + zap.Int64("new_value", memLimitBytes), + zap.Int64("old_value", previous)) + } +} diff --git a/config/example/node.env b/config/example/node.env index 089021767..3abb744be 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -188,3 +188,5 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" + +FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/example/node.json b/config/example/node.json index e4b85bc81..6c98903f1 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -245,5 +245,8 @@ "enabled": true, "endpoint": "localhost:9090", "exporter": "otlp_grpc" + }, + "runtime": { + "soft_memory_limit": 1073741824 } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 897f4e15b..0ef5fea7f 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -217,3 +217,6 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost" + +runtime: + soft_memory_limit: 1gb diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4469b1e10..439edf598 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -24,6 +24,7 @@ There are some custom types used for brevity: | `policer` | [Policer service configuration](#policer-section) | | `replicator` | [Replicator service configuration](#replicator-section) | | `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # `control` section @@ -426,3 +427,15 @@ object: | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | | `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | + +# `runtime` section +Contains runtime parameters. + +```yaml +runtime: + soft_memory_limit: 1GB +``` + +| Parameter | Type | Default value | Description | +|---------------------|--------|---------------|--------------------------------------------------------------------------| +| `soft_memory_limit` | `size` | 0 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a2ff8dcb9..a400187cc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -493,4 +493,6 @@ const ( FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" ) -- 2.45.2