From 37b83c0856fadf10bf13c65b4f3a61c33bafcea0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 Jul 2024 17:46:02 +0300 Subject: [PATCH 01/21] [#1271] getSvc: Fix `head --raw` assemble for EC Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assembleec.go | 4 +- pkg/services/object/get/assemblerec.go | 64 -------------------------- pkg/services/object/get/get.go | 4 ++ pkg/services/object/get/remote.go | 18 ++++++-- 4 files changed, 20 insertions(+), 70 deletions(-) diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 7bbd9ca1e..a58602bf7 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -11,7 +11,7 @@ import ( ) func (r *request) assembleEC(ctx context.Context) { - if r.isRaw() && r.isLocal() { + if r.isRaw() { r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -43,7 +43,7 @@ func (r *request) assembleEC(ctx context.Context) { } r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.isRaw(), r.traverserGenerator, r.curProcEpoch) + assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.traverserGenerator, r.curProcEpoch) r.log.Debug(logs.GetAssemblingECObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index a4021ee5e..d64984d5c 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -37,7 +37,6 @@ type assemblerec struct { cs container.Source log *logger.Logger head bool - raw bool traverserGenerator traverserGenerator epoch uint64 } @@ -51,7 +50,6 @@ func newAssemblerEC( cs container.Source, log *logger.Logger, head bool, - raw bool, tg traverserGenerator, epoch uint64, ) *assemblerec { @@ -64,7 +62,6 @@ func newAssemblerEC( cs: cs, log: log, head: head, - raw: raw, traverserGenerator: tg, epoch: epoch, } @@ -74,9 +71,6 @@ func newAssemblerEC( // It returns parent object. func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { switch { - case a.raw: - err := a.reconstructRawError(ctx) - return nil, err case a.head: return a.reconstructHeader(ctx, writer) case a.rng != nil: @@ -149,56 +143,6 @@ func (a *assemblerec) reconstructObjectFromParts(ctx context.Context, headers bo return c.Reconstruct(parts) } -func (a *assemblerec) reconstructRawError(ctx context.Context) error { - chunks := make(map[string]objectSDK.ECChunk) - var chunksGuard sync.Mutex - for _, ch := range a.ecInfo.localChunks { - chunks[string(ch.ID.GetValue())] = ch - } - - objID := a.addr.Object() - trav, _, err := a.traverserGenerator.GenerateTraverser(a.addr.Container(), &objID, a.epoch) - if err != nil { - return err - } - - eg, ctx := errgroup.WithContext(ctx) - for { - batch := trav.Next() - if len(batch) == 0 { - break - } - for _, node := range batch { - var info client.NodeInfo - client.NodeInfoFromNetmapElement(&info, node) - eg.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if _, found := a.ecInfo.remoteChunks[string(info.PublicKey())]; found { - return nil - } - - nodeChunks := a.tryGetChunkListFromNode(ctx, info) - - chunksGuard.Lock() - defer chunksGuard.Unlock() - for _, ch := range nodeChunks { - chunks[string(ch.ID.GetValue())] = ch - } - return nil - }) - } - } - if err = eg.Wait(); err != nil { - return err - } - return createECInfoErr(chunks) -} - func (a *assemblerec) retrieveParts(ctx context.Context, trav *placement.Traverser, cnr *container.Container) []*objectSDK.Object { dataCount := policy.ECDataCount(cnr.Value.PlacementPolicy()) parityCount := policy.ECParityCount(cnr.Value.PlacementPolicy()) @@ -359,11 +303,3 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli } return object } - -func createECInfoErr(chunks map[string]objectSDK.ECChunk) *objectSDK.ECInfoError { - info := objectSDK.NewECInfo() - for _, ch := range chunks { - info.AddChunk(ch) - } - return objectSDK.NewECInfoError(info) -} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 5a57bc56e..07a2f3a72 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -111,6 +111,10 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: exec.log.Debug(logs.GetRequestedObjectIsEC) + if exec.isRaw() && execCnr { + exec.executeOnContainer(ctx) + exec.analyzeStatus(ctx, false) + } exec.assembleEC(ctx) default: exec.log.Debug(logs.OperationFinishedWithError, diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 4dee15242..ce9abfe1c 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -35,8 +35,12 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { switch { default: r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) - r.status = statusUndefined - r.err = new(apistatus.ObjectNotFound) + if r.status != statusEC { + // for raw requests, continue to collect other parts + r.status = statusUndefined + r.err = new(apistatus.ObjectNotFound) + } + return false case err == nil: r.status = statusOK r.err = nil @@ -48,22 +52,28 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.collectedObject = obj r.writeCollectedObject(ctx) } + return true case errors.As(err, &errRemoved): r.status = statusINHUMED r.err = errRemoved + return true case errors.As(err, &errOutOfRange): r.status = statusOutOfRange r.err = errOutOfRange + return true case errors.As(err, &errSplitInfo): r.status = statusVIRTUAL mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) r.err = objectSDK.NewSplitInfoError(r.infoSplit) + return true case errors.As(err, &errECInfo): r.status = statusEC r.err = r.infoEC.addRemote(string(info.PublicKey()), errECInfo.ECInfo()) + if r.isRaw() { + return false // continue to collect all parts + } + return true } - - return r.status != statusUndefined } func (r *request) getRemote(ctx context.Context, rs remoteStorage, info client.NodeInfo) (*objectSDK.Object, error) { From 308da7cb01f9d885bfde1cd03068d78e440b63d9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 Jul 2024 22:15:13 +0300 Subject: [PATCH 02/21] [#1271] getSvc: Fix local EC chunk get Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemblerec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index d64984d5c..6a02673c3 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -237,7 +237,7 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object return nil } var addr oid.Address - addr.SetContainer(addr.Container()) + addr.SetContainer(a.addr.Container()) addr.SetObject(objID) var object *objectSDK.Object if a.head { From 4d102b05e53ae3f2a6eea1308de7cddae0b7c7b4 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 25 Jul 2024 22:52:48 +0300 Subject: [PATCH 03/21] [#1274] go.mod: Update neo-go version that fixes ws-client * Update go.mod; * This neo-go package version contains fix for the wsclient that allows to morph event listener refresh the invalidated websocket connection to neo-go. Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ee8b1bb16..8e81a6517 100644 --- a/go.mod +++ b/go.mod @@ -128,4 +128,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 +replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 diff --git a/go.sum b/go.sum index c7c3b87eb..7e94195c5 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 h1:O3F2Grz07RWZ68mRz1xsYsNPNvZLwY00BM+xoYb1kNk= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 32ec421ac783ecf2e1432e53063bb4cf1d246d9e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 26 Jul 2024 16:37:05 +0300 Subject: [PATCH 04/21] [#1277] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 23 ---- cmd/frostfs-cli/modules/container/root.go | 3 - cmd/frostfs-cli/modules/container/set_eacl.go | 108 ------------------ cmd/frostfs-cli/modules/container/util.go | 5 +- go.mod | 8 +- go.sum | 12 +- pkg/core/client/client.go | 1 - pkg/morph/client/container/load.go | 40 ------- pkg/network/cache/multi.go | 9 -- .../transport/container/grpc/service.go | 30 ----- pkg/services/container/ape.go | 21 ---- pkg/services/container/ape_test.go | 90 --------------- pkg/services/container/audit.go | 30 ----- pkg/services/container/executor.go | 19 --- pkg/services/container/morph/executor.go | 6 - pkg/services/container/server.go | 2 - pkg/services/container/sign.go | 18 --- 17 files changed, 12 insertions(+), 413 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/container/set_eacl.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index a6d9968c5..215490dbe 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -214,29 +214,6 @@ func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { return } -// SetEACLPrm groups parameters of SetEACL operation. -type SetEACLPrm struct { - Client *client.Client - ClientParams client.PrmContainerSetEACL -} - -// SetEACLRes groups the resulting values of SetEACL operation. -type SetEACLRes struct{} - -// SetEACL requests to save an eACL table in FrostFS. -// -// Operation is asynchronous and no guaranteed even in the absence of errors. -// The required time is also not predictable. -// -// Success can be verified by reading by container identifier. -// -// Returns any error which prevented the operation from completing correctly in error return. -func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.Client.ContainerSetEACL(ctx, prm.ClientParams) - - return -} - // NetworkInfoPrm groups parameters of NetworkInfo operation. type NetworkInfoPrm struct { Client *client.Client diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index 99d1a4231..d5f0fd776 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -26,7 +26,6 @@ func init() { listContainerObjectsCmd, getContainerInfoCmd, getExtendedACLCmd, - setExtendedACLCmd, containerNodesCmd, policyPlaygroundCmd, } @@ -39,7 +38,6 @@ func init() { initContainerListObjectsCmd() initContainerInfoCmd() initContainerGetEACLCmd() - initContainerSetEACLCmd() initContainerNodesCmd() initContainerPolicyPlaygroundCmd() @@ -53,7 +51,6 @@ func init() { }{ {createContainerCmd, "PUT"}, {deleteContainerCmd, "DELETE"}, - {setExtendedACLCmd, "SETEACL"}, } { commonflags.InitSession(el.cmd, "container "+el.verb) } diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go deleted file mode 100644 index 86aa50a57..000000000 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ /dev/null @@ -1,108 +0,0 @@ -package container - -import ( - "bytes" - "errors" - "time" - - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "github.com/spf13/cobra" -) - -var flagVarsSetEACL struct { - noPreCheck bool - - srcPath string -} - -var setExtendedACLCmd = &cobra.Command{ - Use: "set-eacl", - Short: "Set new extended ACL table for container", - Long: `Set new extended ACL table for container. -Container ID in EACL table will be substituted with ID from the CLI.`, - Run: func(cmd *cobra.Command, _ []string) { - id := parseContainerID(cmd) - eaclTable := common.ReadEACL(cmd, flagVarsSetEACL.srcPath) - - tok := getSession(cmd) - - eaclTable.SetCID(id) - - pk := key.GetOrGenerate(cmd) - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - - if !flagVarsSetEACL.noPreCheck { - cmd.Println("Checking the ability to modify access rights in the container...") - - extendable, err := internalclient.IsACLExtendable(cmd.Context(), cli, id) - commonCmd.ExitOnErr(cmd, "Extensibility check failure: %w", err) - - if !extendable { - commonCmd.ExitOnErr(cmd, "", errors.New("container ACL is immutable")) - } - - cmd.Println("ACL extension is enabled in the container, continue processing.") - } - - setEACLPrm := internalclient.SetEACLPrm{ - Client: cli, - ClientParams: client.PrmContainerSetEACL{ - Table: eaclTable, - Session: tok, - }, - } - - _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - if containerAwait { - exp, err := eaclTable.Marshal() - commonCmd.ExitOnErr(cmd, "broken EACL table: %w", err) - - cmd.Println("awaiting...") - - getEACLPrm := internalclient.EACLPrm{ - Client: cli, - ClientParams: client.PrmContainerEACL{ - ContainerID: &id, - }, - } - - for i := 0; i < awaitTimeout; i++ { - time.Sleep(1 * time.Second) - - res, err := internalclient.EACL(cmd.Context(), getEACLPrm) - if err == nil { - // compare binary values because EACL could have been set already - table := res.EACL() - got, err := table.Marshal() - if err != nil { - continue - } - - if bytes.Equal(exp, got) { - cmd.Println("EACL has been persisted on sidechain") - return - } - } - } - - commonCmd.ExitOnErr(cmd, "", errSetEACLTimeout) - } - }, -} - -func initContainerSetEACLCmd() { - commonflags.Init(setExtendedACLCmd) - - flags := setExtendedACLCmd.Flags() - flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - flags.StringVar(&flagVarsSetEACL.srcPath, "table", "", "path to file with JSON or binary encoded EACL table") - flags.BoolVar(&containerAwait, "await", false, "block execution until EACL is persisted") - flags.BoolVar(&flagVarsSetEACL.noPreCheck, "no-precheck", false, "do not pre-check the extensibility of the container ACL") -} diff --git a/cmd/frostfs-cli/modules/container/util.go b/cmd/frostfs-cli/modules/container/util.go index 48265f785..4cb268ec5 100644 --- a/cmd/frostfs-cli/modules/container/util.go +++ b/cmd/frostfs-cli/modules/container/util.go @@ -18,9 +18,8 @@ const ( ) var ( - errCreateTimeout = errors.New("timeout: container has not been persisted on sidechain") - errDeleteTimeout = errors.New("timeout: container has not been removed from sidechain") - errSetEACLTimeout = errors.New("timeout: EACL has not been persisted on sidechain") + errCreateTimeout = errors.New("timeout: container has not been persisted on sidechain") + errDeleteTimeout = errors.New("timeout: container has not been removed from sidechain") ) func parseContainerID(cmd *cobra.Command) cid.ID { diff --git a/go.mod b/go.mod index 8e81a6517..a5504c3e8 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 @@ -23,7 +23,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 - github.com/nspcc-dev/neo-go v0.106.0 + github.com/nspcc-dev/neo-go v0.106.2 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/paulmach/orb v0.11.0 @@ -38,7 +38,7 @@ require ( go.opentelemetry.io/otel v1.22.0 go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/term v0.18.0 diff --git a/go.sum b/go.sum index 7e94195c5..b258f79bf 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e h1:gEWT+70E/RvGkxtSv+PlyUN2vtJVymhQa1mypvrXukM= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de h1:OjsWY0jpGJV1t87XgwL/3PsDx7fJ6lfNMXtY8UhoUbM= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec h1:A09Swh7yogmmiABUf7Ht6MTQXJ07MyGx4+ziUQNelec= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec/go.mod h1:DlJmgV4/qkFkx2ab+YWznlMijiF2yZHnrJswJOB7XGs= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= @@ -321,8 +321,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 8c92901f2..854fbc49f 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -11,7 +11,6 @@ import ( // Client is an interface of FrostFS storage // node's client. type Client interface { - ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) ObjectPutSingle(context.Context, client.PrmObjectPutSingle) (*client.ResObjectPutSingle, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) diff --git a/pkg/morph/client/container/load.go b/pkg/morph/client/container/load.go index b5263d7a6..5e2c3c2c3 100644 --- a/pkg/morph/client/container/load.go +++ b/pkg/morph/client/container/load.go @@ -1,53 +1,13 @@ package container import ( - "crypto/sha256" "fmt" v2refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) -// AnnounceLoadPrm groups parameters of AnnounceLoad operation. -type AnnounceLoadPrm struct { - a container.SizeEstimation - key []byte - - client.InvokePrmOptional -} - -// SetAnnouncement sets announcement. -func (a2 *AnnounceLoadPrm) SetAnnouncement(a container.SizeEstimation) { - a2.a = a -} - -// SetReporter sets public key of the reporter. -func (a2 *AnnounceLoadPrm) SetReporter(key []byte) { - a2.key = key -} - -// AnnounceLoad saves container size estimation calculated by storage node -// with key in FrostFS system through Container contract call. -// -// Returns any error encountered that caused the saving to interrupt. -func (c *Client) AnnounceLoad(p AnnounceLoadPrm) error { - binCnr := make([]byte, sha256.Size) - p.a.Container().Encode(binCnr) - - prm := client.InvokePrm{} - prm.SetMethod(putSizeMethod) - prm.SetArgs(p.a.Epoch(), binCnr, p.a.Value(), p.key) - prm.InvokePrmOptional = p.InvokePrmOptional - - _, err := c.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", putSizeMethod, err) - } - return nil -} - // EstimationID is an identity of container load estimation inside Container contract. type EstimationID []byte diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index f19510d76..9305c143b 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -239,15 +239,6 @@ func (x *multiClient) ObjectPutSingle(ctx context.Context, p client.PrmObjectPut return } -func (x *multiClient) ContainerAnnounceUsedSpace(ctx context.Context, prm client.PrmAnnounceSpace) (res *client.ResAnnounceSpace, err error) { - err = x.iterateClients(ctx, func(c clientcore.Client) error { - res, err = c.ContainerAnnounceUsedSpace(ctx, prm) - return err - }) - - return -} - func (x *multiClient) ObjectDelete(ctx context.Context, p client.PrmObjectDelete) (res *client.ResObjectDelete, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ObjectDelete(ctx, p) diff --git a/pkg/network/transport/container/grpc/service.go b/pkg/network/transport/container/grpc/service.go index ed514d6d4..f0206dd5c 100644 --- a/pkg/network/transport/container/grpc/service.go +++ b/pkg/network/transport/container/grpc/service.go @@ -81,21 +81,6 @@ func (s *Server) List(ctx context.Context, req *containerGRPC.ListRequest) (*con return resp.ToGRPCMessage().(*containerGRPC.ListResponse), nil } -// SetExtendedACL converts gRPC SetExtendedACLRequest message and passes it to internal Container service. -func (s *Server) SetExtendedACL(ctx context.Context, req *containerGRPC.SetExtendedACLRequest) (*containerGRPC.SetExtendedACLResponse, error) { - setEACLReq := new(container.SetExtendedACLRequest) - if err := setEACLReq.FromGRPCMessage(req); err != nil { - return nil, err - } - - resp, err := s.srv.SetExtendedACL(ctx, setEACLReq) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*containerGRPC.SetExtendedACLResponse), nil -} - // GetExtendedACL converts gRPC GetExtendedACLRequest message and passes it to internal Container service. func (s *Server) GetExtendedACL(ctx context.Context, req *containerGRPC.GetExtendedACLRequest) (*containerGRPC.GetExtendedACLResponse, error) { getEACLReq := new(container.GetExtendedACLRequest) @@ -110,18 +95,3 @@ func (s *Server) GetExtendedACL(ctx context.Context, req *containerGRPC.GetExten return resp.ToGRPCMessage().(*containerGRPC.GetExtendedACLResponse), nil } - -// AnnounceUsedSpace converts gRPC AnnounceUsedSpaceRequest message and passes it to internal Container service. -func (s *Server) AnnounceUsedSpace(ctx context.Context, req *containerGRPC.AnnounceUsedSpaceRequest) (*containerGRPC.AnnounceUsedSpaceResponse, error) { - announceReq := new(container.AnnounceUsedSpaceRequest) - if err := announceReq.FromGRPCMessage(req); err != nil { - return nil, err - } - - resp, err := s.srv.AnnounceUsedSpace(ctx, announceReq) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*containerGRPC.AnnounceUsedSpaceResponse), nil -} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 3ea591c6a..8fe4dd2d9 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -78,15 +78,6 @@ func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm } } -func (ac *apeChecker) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.AnnounceUsedSpace") - defer span.End() - - // this method is not used, so not checked - - return ac.next.AnnounceUsedSpace(ctx, req) -} - func (ac *apeChecker) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Delete") defer span.End() @@ -303,18 +294,6 @@ func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.R return nativeschema.PropertyValueContainerRoleOthers, pk, nil } -func (ac *apeChecker) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.SetExtendedACL") - defer span.End() - - if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetEACL().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), - nativeschema.MethodSetContainerEACL); err != nil { - return nil, err - } - - return ac.next.SetExtendedACL(ctx, req) -} - func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, containerID *refs.ContainerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, op string) error { if vh == nil { return errMissingVerificationHeader diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index a6f0fb222..9eed469ca 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -9,7 +9,6 @@ import ( "net" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -50,7 +49,6 @@ func TestAPE(t *testing.T) { t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) t.Run("deny get container by IP", testDenyGetContainerByIP) t.Run("deny get container by group id", testDenyGetContainerByGroupID) - t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) t.Run("deny put container, read namespace from frostfsID", testDenyPutContainerReadNamespaceFromFrostfsID) @@ -665,84 +663,6 @@ func testDenyGetContainerByGroupID(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } -func testDenySetContainerEACLForIR(t *testing.T) { - t.Parallel() - srv := &srvStub{ - calls: map[string]int{}, - } - router := inmemory.NewInMemory() - contRdr := &containerStub{ - c: map[cid.ID]*containercore.Container{}, - } - ir := &irStub{ - keys: [][]byte{}, - } - nm := &netmapStub{} - frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, - } - apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) - - contID := cidtest.ID() - testContainer := containertest.Container() - pp := netmap.PlacementPolicy{} - require.NoError(t, pp.DecodeString("REP 1")) - testContainer.SetPlacementPolicy(pp) - contRdr.c[contID] = &containercore.Container{Value: testContainer} - - nm.currentEpoch = 100 - nm.netmaps = map[uint64]*netmap.NetMap{} - var testNetmap netmap.NetMap - testNetmap.SetEpoch(nm.currentEpoch) - testNetmap.SetNodes([]netmap.NodeInfo{{}}) - nm.netmaps[nm.currentEpoch] = &testNetmap - nm.netmaps[nm.currentEpoch-1] = &testNetmap - - _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ - Rules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{ - Names: []string{ - nativeschema.MethodSetContainerEACL, - }, - }, - Resources: chain.Resources{ - Names: []string{ - fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), - }, - }, - Condition: []chain.Condition{ - { - Kind: chain.KindRequest, - Key: nativeschema.PropertyKeyActorRole, - Value: nativeschema.PropertyValueContainerRoleIR, - Op: chain.CondStringEquals, - }, - }, - }, - }, - }) - require.NoError(t, err) - - req := &container.SetExtendedACLRequest{} - req.SetBody(&container.SetExtendedACLRequestBody{}) - var refContID refs.ContainerID - contID.WriteToV2(&refContID) - req.GetBody().SetEACL(&acl.Table{}) - req.GetBody().GetEACL().SetContainerID(&refContID) - - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) - ir.keys = append(ir.keys, pk.PublicKey().Bytes()) - - resp, err := apeSrv.SetExtendedACL(context.Background(), req) - require.Nil(t, resp) - var errAccessDenied *apistatus.ObjectAccessDenied - require.ErrorAs(t, err, &errAccessDenied) -} - func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -1229,11 +1149,6 @@ type srvStub struct { calls map[string]int } -func (s *srvStub) AnnounceUsedSpace(context.Context, *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - s.calls["AnnounceUsedSpace"]++ - return &container.AnnounceUsedSpaceResponse{}, nil -} - func (s *srvStub) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) { s.calls["Delete"]++ return &container.DeleteResponse{}, nil @@ -1259,11 +1174,6 @@ func (s *srvStub) Put(context.Context, *container.PutRequest) (*container.PutRes return &container.PutResponse{}, nil } -func (s *srvStub) SetExtendedACL(context.Context, *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - s.calls["SetExtendedACL"]++ - return &container.SetExtendedACLResponse{}, nil -} - type irStub struct { keys [][]byte } diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index 7ef432bb1..34fd5923f 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" container_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -29,24 +28,6 @@ func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Serv } } -// AnnounceUsedSpace implements Server. -func (a *auditService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - res, err := a.next.AnnounceUsedSpace(ctx, req) - if !a.enabled.Load() { - return res, err - } - - var ids []*refs.ContainerID - for _, v := range req.GetBody().GetAnnouncements() { - ids = append(ids, v.GetContainerID()) - } - - audit.LogRequest(a.log, container_grpc.ContainerService_AnnounceUsedSpace_FullMethodName, req, - audit.TargetFromRefs(ids, &cid.ID{}), err == nil) - - return res, err -} - // Delete implements Server. func (a *auditService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { res, err := a.next.Delete(ctx, req) @@ -103,14 +84,3 @@ func (a *auditService) Put(ctx context.Context, req *container.PutRequest) (*con audit.TargetFromRef(res.GetBody().GetContainerID(), &cid.ID{}), err == nil) return res, err } - -// SetExtendedACL implements Server. -func (a *auditService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - res, err := a.next.SetExtendedACL(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, container_grpc.ContainerService_SetExtendedACL_FullMethodName, req, - audit.TargetFromRef(req.GetBody().GetEACL().GetContainerID(), &cid.ID{}), err == nil) - return res, err -} diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index d4ae11d62..b64963e25 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -14,7 +14,6 @@ type ServiceExecutor interface { Delete(context.Context, *session.Token, *container.DeleteRequestBody) (*container.DeleteResponseBody, error) Get(context.Context, *container.GetRequestBody) (*container.GetResponseBody, error) List(context.Context, *container.ListRequestBody) (*container.ListResponseBody, error) - SetExtendedACL(context.Context, *session.Token, *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) GetExtendedACL(context.Context, *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) } @@ -96,24 +95,6 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co return resp, nil } -func (s *executorSvc) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - meta := req.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - respBody, err := s.exec.SetExtendedACL(ctx, meta.GetSessionToken(), req.GetBody()) - if err != nil { - return nil, fmt.Errorf("could not execute SetEACL request: %w", err) - } - - resp := new(container.SetExtendedACLResponse) - resp.SetBody(respBody) - - s.respSvc.SetMeta(resp) - return resp, nil -} - func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { respBody, err := s.exec.GetExtendedACL(ctx, req.GetBody()) if err != nil { diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index e2e79f3d2..57dac32f0 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -13,8 +13,6 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) var errMissingUserID = errors.New("missing user ID") @@ -204,10 +202,6 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } -func (s *morphExecutor) SetExtendedACL(_ context.Context, _ *sessionV2.Token, _ *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { - return nil, status.Errorf(codes.Unimplemented, "method SetExtendedACL not implemented") -} - func (s *morphExecutor) GetExtendedACL(_ context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { diff --git a/pkg/services/container/server.go b/pkg/services/container/server.go index 052a8c945..d714d7f02 100644 --- a/pkg/services/container/server.go +++ b/pkg/services/container/server.go @@ -12,7 +12,5 @@ type Server interface { Get(context.Context, *container.GetRequest) (*container.GetResponse, error) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) List(context.Context, *container.ListRequest) (*container.ListResponse, error) - SetExtendedACL(context.Context, *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) GetExtendedACL(context.Context, *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) - AnnounceUsedSpace(context.Context, *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index bba717f60..62aa3fe27 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -57,15 +57,6 @@ func (s *signService) List(ctx context.Context, req *container.ListRequest) (*co return resp, s.sigSvc.SignResponse(resp, err) } -func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - if err := s.sigSvc.VerifyRequest(req); err != nil { - resp := new(container.SetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(resp, err) - } - resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(resp, err) -} - func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetExtendedACLResponse) @@ -74,12 +65,3 @@ func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExte resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) return resp, s.sigSvc.SignResponse(resp, err) } - -func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - if err := s.sigSvc.VerifyRequest(req); err != nil { - resp := new(container.AnnounceUsedSpaceResponse) - return resp, s.sigSvc.SignResponse(resp, err) - } - resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) - return resp, s.sigSvc.SignResponse(resp, err) -} From 1b92817bd38c6e9e15cb49bd058ecb8f07cfffdb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 Jul 2024 11:31:05 +0300 Subject: [PATCH 05/21] [#1278] ir: Do not allow to create container without FrostFSID record Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/container/handlers_test.go | 3 +-- .../processors/container/process_container.go | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 1aac31ae3..dc1e919bb 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -3,7 +3,6 @@ package container import ( "crypto/ecdsa" "encoding/hex" - "fmt" "testing" "time" @@ -238,5 +237,5 @@ func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) type testFrostFSIDClient struct{} func (c *testFrostFSIDClient) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { - return nil, fmt.Errorf("subject not found") + return &frostfsidclient.Subject{}, nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index a950997fd..d89b63e82 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -180,11 +180,6 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain } } - namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") - if !hasNamespace { - return nil - } - addr, err := util.Uint160DecodeBytesBE(cnr.Owner().WalletBytes()[1 : 1+util.Uint160Size]) if err != nil { return fmt.Errorf("could not get container owner address: %w", err) @@ -195,6 +190,11 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain return fmt.Errorf("could not get subject from FrostfsID contract: %w", err) } + namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") + if !hasNamespace { + return nil + } + if subject.Namespace != namespace { return errContainerAndOwnerNamespaceDontMatch } From f53d30fa954617cc0fbf6029cb5d606da551954f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 Jul 2024 13:03:55 +0300 Subject: [PATCH 06/21] [#1278] containerSvc: Validate FrostFSID subject exitence on Put Signed-off-by: Dmitrii Stepanov --- pkg/services/container/ape.go | 21 ++++++++++++++++++++- pkg/services/container/ape_test.go | 15 ++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 8fe4dd2d9..6f8a8e0e6 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -211,7 +211,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont } } - namespace, err := ac.namespaceByOwner(req.GetBody().GetContainer().GetOwnerID()) + namespace, err := ac.namespaceByKnownOwner(req.GetBody().GetContainer().GetOwnerID()) if err != nil { return nil, fmt.Errorf("get namespace error: %w", err) } @@ -608,6 +608,25 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { return namespace, nil } +func (ac *apeChecker) namespaceByKnownOwner(owner *refs.OwnerID) (string, error) { + var ownerSDK user.ID + if owner == nil { + return "", errOwnerIDIsNotSet + } + if err := ownerSDK.ReadFromV2(*owner); err != nil { + return "", err + } + addr, err := ownerSDK.ScriptHash() + if err != nil { + return "", err + } + subject, err := ac.frostFSIDClient.GetSubject(addr) + if err != nil { + return "", fmt.Errorf("get subject error: %w", err) + } + return subject.Namespace, nil +} + // validateNamespace validates a namespace set in a container. // If frostfs-id contract stores a namespace N1 for an owner ID and a container within a request // is set with namespace N2 (via Zone() property), then N2 is invalid and the request is denied. diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 9eed469ca..68c1158a6 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -765,17 +765,22 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, - } - apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) testContainer := containertest.Container() + owner := testContainer.Owner() + ownerAddr, err := owner.ScriptHash() + require.NoError(t, err) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + ownerAddr: {}, + }, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) nm.currentEpoch = 100 nm.netmaps = map[uint64]*netmap.NetMap{} - _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ Rules: []chain.Rule{ { Status: chain.AccessDenied, From cecb3f786761b966d25022475aba81e774f87e48 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jul 2024 15:41:57 +0300 Subject: [PATCH 07/21] [#1282] cli: Allow to external addresses first for `object nodes` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index d04cf6f04..42ae7324e 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -30,7 +30,8 @@ import ( ) const ( - verifyPresenceAllFlag = "verify-presence-all" + verifyPresenceAllFlag = "verify-presence-all" + preferInternalAddressesFlag = "prefer-internal-addresses" ) var ( @@ -97,6 +98,7 @@ func initObjectNodesCmd() { flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes.") flags.Bool(commonflags.JSON, false, "Print information about the object placement as json.") + flags.Bool(preferInternalAddressesFlag, false, "Use internal addresses first to get object info.") } func objectNodes(cmd *cobra.Command, _ []string) { @@ -449,11 +451,20 @@ func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { var cli *client.Client var addresses []string - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) - addresses = append(addresses, candidate.ExternalAddresses()...) + if preferInternal, _ := cmd.Flags().GetBool(preferInternalAddressesFlag); preferInternal { + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + addresses = append(addresses, candidate.ExternalAddresses()...) + } else { + addresses = append(addresses, candidate.ExternalAddresses()...) + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + } + var lastErr error for _, address := range addresses { var networkAddr network.Address From 02d191e9a6a1c347ae95f81ba6b9e125dde9efef Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 29 Jul 2024 15:33:36 +0300 Subject: [PATCH 08/21] [#1279] adm: Interpret "root" name as empty for namespace target type Signed-off-by: Airat Arifullin --- cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 5e17f4014..d4aedda2e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -38,6 +38,12 @@ var ( func parseTarget(cmd *cobra.Command) policyengine.Target { name, _ := cmd.Flags().GetString(targetNameFlag) typ, err := parseTargetType(cmd) + + // interpret "root" namespace as empty + if typ == policyengine.Namespace && name == "root" { + name = "" + } + commonCmd.ExitOnErr(cmd, "read target type error: %w", err) return policyengine.Target{ From 20f308876c7e80dea63c891d3f52f52ffcc50778 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 1 Aug 2024 16:08:50 +0300 Subject: [PATCH 09/21] [#1288] putSvc: Respect TTL for EC put Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/ec.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index 6da50195e..fbb51912c 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -17,6 +17,7 @@ import ( containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -39,7 +40,7 @@ type ecWriter struct { } func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { - relayed, err := e.relayIfNotContainerNode(ctx) + relayed, err := e.relayIfNotContainerNode(ctx, obj) if err != nil { return err } @@ -65,7 +66,7 @@ func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error return e.writeRawObject(ctx, obj) } -func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { +func (e *ecWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, error) { if e.relay == nil { return false, nil } @@ -77,7 +78,13 @@ func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { // object can be splitted or saved local return false, nil } - if err := e.relayToContainerNode(ctx); err != nil { + objID := object.AddressOf(obj).Object() + var index uint32 + if obj.ECHeader() != nil { + objID = obj.ECHeader().Parent() + index = obj.ECHeader().Index() + } + if err := e.relayToContainerNode(ctx, objID, index); err != nil { return false, err } return true, nil @@ -102,18 +109,20 @@ func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { return false, nil } -func (e *ecWriter) relayToContainerNode(ctx context.Context) error { - t, err := placement.NewTraverser(e.placementOpts...) +func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index uint32) error { + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(objID))...) if err != nil { return err } var lastErr error + offset := int(index) for { nodes := t.Next() if len(nodes) == 0 { break } - for _, node := range nodes { + for idx := range nodes { + node := nodes[(idx+offset)%len(nodes)] var info client.NodeInfo client.NodeInfoFromNetmapElement(&info, node) @@ -149,6 +158,10 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context) error { } func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { + if e.commonPrm.LocalOnly() { + return e.writePartLocal(ctx, obj) + } + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(obj.ECHeader().Parent()))...) if err != nil { return err From 28fc41bd982c26831ce3a79c27eb9103a1f7ffb3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Aug 2024 17:50:49 +0300 Subject: [PATCH 10/21] [#1291] morph: Reconnect to the highest priority endpoint Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/constructor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 648c7d3c0..78cb3e82f 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -148,6 +148,10 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } else { cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, zap.String("endpoint", endpoint.Address)) + if cli.endpoints.curr > 0 && cli.cfg.switchInterval != 0 { + cli.switchIsActive.Store(true) + go cli.switchToMostPrioritized(ctx) + } break } } From 87c5954f4e0fd06585cae8905a312ceab3e2e4c7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Aug 2024 13:24:48 +0300 Subject: [PATCH 11/21] [#1295] engine: Log object address in case of error Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/delete.go | 6 +++--- pkg/local_object_storage/engine/exists.go | 3 ++- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 3 ++- pkg/local_object_storage/engine/inhume.go | 8 ++++---- pkg/local_object_storage/engine/lock.go | 13 +++++++++---- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- 8 files changed, 23 insertions(+), 16 deletions(-) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 096528967..318f938fb 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -100,7 +100,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e return false } else { if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check object existence", err) + e.reportShardError(sh, "could not check object existence", err, zap.Stringer("address", prm.addr)) } return false } @@ -116,7 +116,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e _, err = sh.Inhume(ctx, shPrm) if err != nil { - e.reportShardError(sh, "could not inhume object in shard", err) + e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", prm.addr)) var target *apistatus.ObjectLocked locked.is = errors.As(err, &target) @@ -191,7 +191,7 @@ func (e *StorageEngine) deleteChunks( var objID oid.ID err := objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not delete EC chunk", err) + e.reportShardError(sh, "could not delete EC chunk", err, zap.Stringer("address", prm.addr)) } addr.SetObject(objID) inhumePrm.MarkAsGarbage(addr) diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index c57f79691..d98101306 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" ) // exists return in the first value true if object exists. @@ -36,7 +37,7 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool } if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check existence of object in shard", err) + e.reportShardError(sh, "could not check existence of object in shard", err, zap.Stringer("address", shPrm.Address)) } return false } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 991af3d1a..253256c34 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -186,7 +186,7 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { i.ObjectExpired = true return true default: - i.Engine.reportShardError(sh, "could not get object from shard", err) + i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 92d1b20fc..dfe5e48a1 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -12,6 +12,7 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" ) // HeadPrm groups the parameters of Head operation. @@ -118,7 +119,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) outError = new(apistatus.ObjectNotFound) return true default: - e.reportShardError(sh, "could not head object from shard", err) + e.reportShardError(sh, "could not head object from shard", err, zap.Stringer("address", prm.addr)) return false } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 991305af0..683713f94 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -154,7 +154,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError if !(errors.As(err, &siErr) || errors.As(err, &ecErr)) { - e.reportShardError(sh, "could not check for presents in shard", err) + e.reportShardError(sh, "could not check for presents in shard", err, zap.Stringer("address", addr)) return } @@ -179,7 +179,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh return true } - e.reportShardError(sh, "could not inhume object in shard", err) + e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", addr)) return false } @@ -205,7 +205,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr), + e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err return false @@ -235,7 +235,7 @@ func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid. e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { ld, err := h.Shard.GetLocked(ctx, addr) if err != nil { - e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("addr", addr), + e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 5354c205f..3a41a7848 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -13,6 +13,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" ) var errLockFailed = errors.New("lock operation failed") @@ -90,14 +91,16 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo var objID oid.ID err = objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } eclocked = append(eclocked, objID) } err = sh.Lock(ctx, idCnr, locker, eclocked) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } root = true @@ -109,7 +112,8 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo return true } - e.reportShardError(sh, "could not check locked object for presence in shard", err) + e.reportShardError(sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return } @@ -121,7 +125,8 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo err := sh.Lock(ctx, idCnr, locker, []oid.ID{locked}) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) var errIrregular *apistatus.LockNonRegularObject if errors.As(err, &errIrregular) { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 54385910b..f92d83745 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -187,7 +187,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti return } - e.reportShardError(sh, "could not put object to shard", err) + e.reportShardError(sh, "could not put object to shard", err, zap.Stringer("address", addr)) return } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index f5b33a251..cbf26ff4e 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -208,7 +208,7 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { return true // stop, return it back default: - i.Engine.reportShardError(sh, "could not get object from shard", err) + i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) From 89d670c2f0df4e23af8acd4d684b5580ed78ebbe Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Aug 2024 16:13:43 +0300 Subject: [PATCH 12/21] [#1295] engine: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/lock.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 3a41a7848..ac8fa9c6f 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -63,11 +63,9 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { // code is pretty similar to inhumeAddr, maybe unify? root := false - var addrLocked oid.Address addrLocked.SetContainer(idCnr) addrLocked.SetObject(locked) - e.iterateOverSortedShards(addrLocked, func(_ int, sh hashedShard) (stop bool) { defer func() { // if object is root we continue since information about it @@ -80,7 +78,6 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo if checkExists { var existsPrm shard.ExistsPrm existsPrm.Address = addrLocked - exRes, err := sh.Exists(ctx, existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError @@ -111,7 +108,6 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo // do not lock it return true } - e.reportShardError(sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return @@ -133,14 +129,10 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo status = 1 return true } - return false } - status = 2 - return true }) - return } From 54862250b2c5d85944ef28ad8a6cb70348540b38 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 09:32:40 +0300 Subject: [PATCH 13/21] [#1295] getSvc: Assemble complex EC object headers without linking object Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/head.go | 15 ++++----- pkg/local_object_storage/metabase/get.go | 22 +++++++++--- pkg/services/object/get/assemble.go | 4 +-- pkg/services/object/get/assembler.go | 43 ++++++++++++++++++++++++ pkg/services/object/get/get_test.go | 2 +- pkg/services/object/get/request.go | 4 +-- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index dfe5e48a1..6857a3631 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -127,17 +127,16 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) return true }) + if head != nil { + return HeadRes{head: head}, nil + } if outSI != nil { return HeadRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) - } else if outEI != nil { - return HeadRes{}, logicerr.Wrap(objectSDK.NewECInfoError(outEI)) - } else if head == nil { - return HeadRes{}, outError } - - return HeadRes{ - head: head, - }, nil + if outEI != nil { + return HeadRes{}, logicerr.Wrap(objectSDK.NewECInfoError(outEI)) + } + return HeadRes{}, outError } // Head reads object header from local storage by provided address. diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index d9acd4ce2..b79f6cb14 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -160,11 +160,23 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } - // pick last item, for now there is not difference which address to pick - // but later list might be sorted so first or last value can be more - // prioritized to choose - virtualOID := relativeLst[len(relativeLst)-1] - data := getFromBucket(tx, primaryBucketName(cnr, bucketName), virtualOID) + var data []byte + for i := 0; i < len(relativeLst) && len(data) == 0; i++ { + virtualOID := relativeLst[len(relativeLst)-i-1] + data = getFromBucket(tx, primaryBucketName(cnr, bucketName), virtualOID) + } + + if len(data) == 0 { + // check if any of the relatives is an EC object + for _, relative := range relativeLst { + data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), relative) + if len(data) > 0 { + // we can't return object headers, but can return error, + // so assembler can try to assemble complex object + return nil, getSplitInfoError(tx, cnr, key) + } + } + } child := objectSDK.New() diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index ba6fddec5..9f17f1e4c 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -12,7 +12,7 @@ import ( ) func (r *request) assemble(ctx context.Context) { - if !r.canAssemble() { + if !r.canAssembleComplexObject() { r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -38,7 +38,7 @@ func (r *request) assemble(ctx context.Context) { r.log.Debug(logs.GetTryingToAssembleTheObject) r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r) + assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r, r.headOnly()) r.log.Debug(logs.GetAssemblingSplittedObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 025296ec7..ff3f90bf2 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -19,6 +19,7 @@ type assembler struct { splitInfo *objectSDK.SplitInfo rng *objectSDK.Range objGetter objectGetter + head bool currentOffset uint64 @@ -30,18 +31,23 @@ func newAssembler( splitInfo *objectSDK.SplitInfo, rng *objectSDK.Range, objGetter objectGetter, + head bool, ) *assembler { return &assembler{ addr: addr, rng: rng, splitInfo: splitInfo, objGetter: objGetter, + head: head, } } // Assemble assembles splitted large object and writes it's content to ObjectWriter. // It returns parent object. func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + if a.head { + return a.assembleHeader(ctx, writer) + } sourceObjectID, ok := a.getLastPartOrLinkObjectID() if !ok { return nil, objectSDK.NewSplitInfoError(a.splitInfo) @@ -65,6 +71,43 @@ func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectS return a.parentObject, nil } +func (a *assembler) assembleHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + var sourceObjectIDs []oid.ID + sourceObjectID, ok := a.splitInfo.Link() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + sourceObjectID, ok = a.splitInfo.LastPart() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + if len(sourceObjectIDs) == 0 { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + for _, sourceObjectID = range sourceObjectIDs { + obj, err := a.getParent(ctx, sourceObjectID, writer) + if err == nil { + return obj, nil + } + } + return nil, objectSDK.NewSplitInfoError(a.splitInfo) +} + +func (a *assembler) getParent(ctx context.Context, sourceObjectID oid.ID, writer ObjectWriter) (*objectSDK.Object, error) { + obj, err := a.objGetter.HeadObject(ctx, sourceObjectID) + if err != nil { + return nil, err + } + parent := obj.Parent() + if parent == nil { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + if err := writer.WriteHeader(ctx, parent); err != nil { + return nil, err + } + return obj, nil +} + func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) { sourceObjectID, ok := a.splitInfo.Link() if ok { diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 29a15ba78..1fc6b7b20 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -730,7 +730,7 @@ func TestGetRemoteSmall(t *testing.T) { t.Run("VIRTUAL", func(t *testing.T) { testHeadVirtual := func(svc *Service, addr oid.Address, i *objectSDK.SplitInfo) { - headPrm := newHeadPrm(false, nil) + headPrm := newHeadPrm(true, nil) headPrm.WithAddress(addr) errSplit := objectSDK.NewSplitInfoError(objectSDK.NewSplitInfo()) diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index 9ddfeddf2..1a7a43a35 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -88,8 +88,8 @@ func (r *request) key() (*ecdsa.PrivateKey, error) { return r.keyStore.GetKey(sessionInfo) } -func (r *request) canAssemble() bool { - return !r.isRaw() && !r.headOnly() +func (r *request) canAssembleComplexObject() bool { + return !r.isRaw() } func (r *request) splitInfo() *objectSDK.SplitInfo { From b3ab9589a566995b3f6f4b9151f74a5f44d56551 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Aug 2024 13:15:22 +0300 Subject: [PATCH 14/21] [#1302] writecache: Add put->flush->put benchmark Signed-off-by: Dmitrii Stepanov --- .../writecache/benchmark/writecache_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index c1c0e88b3..4f4398452 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -2,6 +2,7 @@ package benchmark import ( "context" + "fmt" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -27,6 +28,24 @@ func BenchmarkWritecachePar(b *testing.B) { }) } +func BenchmarkWriteAfterDelete(b *testing.B) { + const payloadSize = 32 << 10 + const parallel = 25 + + cache := newCache(b) + benchmarkPutPrepare(b, cache) + b.Run(fmt.Sprintf("%dB_before", payloadSize), func(b *testing.B) { + b.SetParallelism(parallel) + benchmarkRunPar(b, cache, payloadSize) + }) + require.NoError(b, cache.Flush(context.Background(), false, false)) + b.Run(fmt.Sprintf("%dB_after", payloadSize), func(b *testing.B) { + b.SetParallelism(parallel) + benchmarkRunPar(b, cache, payloadSize) + }) + require.NoError(b, cache.Close()) +} + func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) defer func() { require.NoError(b, cache.Close()) }() @@ -54,6 +73,10 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) defer func() { require.NoError(b, cache.Close()) }() + benchmarkRunPar(b, cache, size) +} + +func benchmarkRunPar(b *testing.B, cache writecache.Cache, size uint64) { ctx := context.Background() b.ResetTimer() From 078b5fdd6e09a63f881d5707edd05306ee434f69 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Aug 2024 13:32:18 +0300 Subject: [PATCH 15/21] [#1302] writecache: Allow to specify custom page size Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/writecache/inspect.go | 2 +- cmd/frostfs-lens/internal/writecache/list.go | 2 +- cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go | 11 +++++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 2 ++ pkg/local_object_storage/writecache/options.go | 9 +++++++++ pkg/local_object_storage/writecache/storage.go | 2 +- pkg/local_object_storage/writecache/util.go | 3 ++- 12 files changed, 36 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index afc986c8b..63c669a35 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -25,7 +25,7 @@ func init() { func inspectFunc(cmd *cobra.Command, _ []string) { var data []byte - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index bcbae0ec9..9c8fa6138 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -31,7 +31,7 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5b91e7819..6b79b9a3f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -153,6 +153,7 @@ type shardCfg struct { flushWorkerCount int sizeLimit uint64 noSync bool + pageSize int } piloramaCfg struct { @@ -271,6 +272,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.path = writeCacheCfg.Path() wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() + wc.pageSize = writeCacheCfg.BoltDB().PageSize() wc.maxObjSize = writeCacheCfg.MaxObjectSize() wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() @@ -863,6 +865,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithPath(wcRead.path), writecache.WithMaxBatchSize(wcRead.maxBatchSize), writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), + writecache.WithPageSize(wcRead.pageSize), writecache.WithMaxObjectSize(wcRead.maxObjSize), writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 7473afefb..ed6f0f694 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -78,6 +78,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) + require.EqualValues(t, 4096, wc.BoltDB().PageSize()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) @@ -133,6 +134,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) + require.EqualValues(t, 0, wc.BoltDB().PageSize()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go index 9e334cd8f..a51308b5b 100644 --- a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go +++ b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go @@ -60,3 +60,14 @@ func (x *Config) MaxBatchSize() int { func (x *Config) NoSync() bool { return config.BoolSafe((*config.Config)(x), "no_sync") } + +// PageSize returns the value of "page_size" config parameter. +// +// Returns 0 if the value is not a positive number. +func (x *Config) PageSize() int { + s := int(config.SizeInBytesSafe((*config.Config)(x), "page_size")) + if s < 0 { + s = 0 + } + return s +} diff --git a/config/example/node.env b/config/example/node.env index 00190eb39..3beb4658c 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_PAGE_SIZE=4096 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index 9051d2bb7..763f7c169 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -148,7 +148,8 @@ "small_object_size": 16384, "max_object_size": 134217728, "flush_worker_count": 30, - "capacity": 3221225472 + "capacity": 3221225472, + "page_size": 4096 }, "metabase": { "path": "tmp/0/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index bcc8552b3..8c5793bac 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -171,6 +171,7 @@ storage: no_sync: true path: tmp/0/cache # write-cache root directory capacity: 3221225472 # approximate write-cache total size, bytes + page_size: 4k metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4a6e5ba6d..0b4fb5431 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -290,6 +290,7 @@ writecache: small_object_size: 16384 max_object_size: 134217728 flush_worker_count: 30 + page_size: '4k' ``` | Parameter | Type | Default value | Description | @@ -301,6 +302,7 @@ writecache: | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | | `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | # `node` section diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index c8eb1bc45..585cf2533 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -45,6 +45,8 @@ type options struct { metrics Metrics // disableBackgroundFlush is for testing purposes only. disableBackgroundFlush bool + // pageSize is bbolt's page size config value + pageSize int } // WithLogger sets logger. @@ -163,3 +165,10 @@ func WithDisableBackgroundFlush() Option { o.disableBackgroundFlush = true } } + +// WithPageSize sets bbolt's page size. +func WithPageSize(s int) Option { + return func(o *options) { + o.pageSize = s + } +} diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index caf997af8..57021cc17 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -32,7 +32,7 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return err } - c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile) + c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile, c.pageSize) if err != nil { return fmt.Errorf("could not open database: %w", err) } diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/util.go index 0ed4a954e..ad3b443f3 100644 --- a/pkg/local_object_storage/writecache/util.go +++ b/pkg/local_object_storage/writecache/util.go @@ -10,11 +10,12 @@ import ( ) // OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { +func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error), pageSize int) (*bbolt.DB, error) { return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ NoFreelistSync: true, ReadOnly: ro, Timeout: 100 * time.Millisecond, OpenFile: openFile, + PageSize: pageSize, }) } From 8b4f3d82c925b65b68579767b8f3694ecf90b6b2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Aug 2024 10:38:45 +0300 Subject: [PATCH 16/21] [#1302] putSvc: Override SuccessAfter for non-regular objects in EC containers Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 4 ++++ pkg/services/object/put/streamer.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 6d2f3dba8..9fa8ddb67 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -19,6 +19,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -229,6 +230,9 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb if len(copiesNumber) > 0 && !result.isEC { result.placementOptions = append(result.placementOptions, placement.WithCopyNumbers(copiesNumber)) } + if container.IsECContainer(cnrInfo.Value) && !object.IsECSupported(obj) && !localOnly { + result.placementOptions = append(result.placementOptions, placement.SuccessAfter(uint32(policy.ECParityCount(cnrInfo.Value.PlacementPolicy())+1))) + } result.placementOptions = append(result.placementOptions, placement.ForContainer(cnrInfo.Value)) objID, ok := obj.ID() diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 14dae38d5..4e655ed54 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -212,10 +213,10 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { if container.IsECContainer(prm.cnr) && object.IsECSupported(prm.hdr) { return p.newECWriter(prm) } - return p.newDefaultObjectWriter(prm) + return p.newDefaultObjectWriter(prm, false) } -func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { +func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm, forECPlacement bool) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -232,9 +233,16 @@ func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWri } } + traverseOpts := prm.traverseOpts + if forECPlacement && !prm.common.LocalOnly() { + // save non-regular and linking object to EC container. + // EC 2.1 -> REP 2, EC 2.2 -> REP 3 etc. + traverseOpts = append(traverseOpts, placement.SuccessAfter(uint32(policy.ECParityCount(prm.cnr.PlacementPolicy())+1))) + } + return &distributedTarget{ cfg: p.cfg, - placementOpts: prm.traverseOpts, + placementOpts: traverseOpts, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ @@ -266,7 +274,7 @@ func (p *Streamer) newECWriter(prm *PutInitPrm) transformer.ObjectWriter { commonPrm: prm.common, relay: p.relay, }, - repWriter: p.newDefaultObjectWriter(prm), + repWriter: p.newDefaultObjectWriter(prm, true), } } From 4c520be9f1fe7a14a28d64914434505543e7eb54 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Aug 2024 15:39:22 +0300 Subject: [PATCH 17/21] [#1313] blobovnicza: Prevent concurrent Put/Close Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobovnicza/blobovnicza.go | 2 +- pkg/local_object_storage/blobovnicza/delete.go | 3 +++ pkg/local_object_storage/blobovnicza/exists.go | 3 +++ pkg/local_object_storage/blobovnicza/get.go | 3 +++ pkg/local_object_storage/blobovnicza/iterate.go | 3 +++ pkg/local_object_storage/blobovnicza/move.go | 3 +++ pkg/local_object_storage/blobovnicza/put.go | 3 +++ 7 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index c787f9d5e..d203ea077 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -22,7 +22,7 @@ type Blobovnicza struct { boltDB *bbolt.DB opened bool - controlMtx sync.Mutex + controlMtx sync.RWMutex } // Option is an option of Blobovnicza's constructor. diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 5d6787897..60438b5c6 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -45,6 +45,9 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + addrKey := addressKey(prm.addr) found := false diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index f7bc84d4a..12a043295 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -21,6 +21,9 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + addrKey := addressKey(addr) err := b.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 600323f55..b73db8178 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -51,6 +51,9 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + var ( data []byte addrKey = addressKey(prm.addr) diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 01e5529da..d52271674 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -128,6 +128,9 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/move.go b/pkg/local_object_storage/blobovnicza/move.go index 420e22a48..3b56f6527 100644 --- a/pkg/local_object_storage/blobovnicza/move.go +++ b/pkg/local_object_storage/blobovnicza/move.go @@ -29,6 +29,9 @@ func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error { )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + key := addressKey(prm.Address) err := b.boltDB.Update(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index ff223ba36..45073d9df 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -64,6 +64,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { )) defer span.End() + b.controlMtx.RLock() + defer b.controlMtx.RUnlock() + sz := uint64(len(prm.objData)) bucketName := bucketForSize(sz) key := addressKey(prm.addr) From f82b7e1ae3759f4cb15d2cca233b5a51fff3f91b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 15 Aug 2024 18:24:10 +0300 Subject: [PATCH 18/21] [#1135] ir: Add healthstatus RECONFIGURING Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-ir/config.go | 6 +++ pkg/innerring/state.go | 12 ++++++ pkg/services/control/ir/service_grpc.pb.go | 6 ++- pkg/services/control/ir/types.pb.go | 18 ++++++--- pkg/services/control/ir/types.proto | 3 ++ pkg/services/tree/service_grpc.pb.go | 44 +++++++++------------- 6 files changed, 54 insertions(+), 35 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 955195477..4eaac845c 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -7,6 +7,7 @@ import ( configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -81,6 +82,10 @@ func watchForSignal(cancel func()) { return case <-sighupCh: log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + if !innerRing.CompareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { + log.Info(logs.FrostFSNodeSIGHUPSkip) + break + } err := reloadConfig() if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) @@ -92,6 +97,7 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + innerRing.CompareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } } diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 1616dbb9f..d3071faad 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -161,6 +161,16 @@ func (s *Server) setHealthStatus(hs control.HealthStatus) { } } +func (s *Server) CompareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { + if swapped = s.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { + s.notifySystemd(newSt) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(newSt)) + } + } + return +} + // HealthStatus returns the current health status of the IR application. func (s *Server) HealthStatus() control.HealthStatus { return control.HealthStatus(s.healthStatus.Load()) @@ -186,6 +196,8 @@ func (s *Server) notifySystemd(st control.HealthStatus) { err = sdnotify.FlagAndStatus(sdnotify.ReadyEnabled) case control.HealthStatus_SHUTTING_DOWN: err = sdnotify.FlagAndStatus(sdnotify.StoppingEnabled) + case control.HealthStatus_RECONFIGURING: + err = sdnotify.FlagAndStatus(sdnotify.ReloadingEnabled) default: err = sdnotify.Status(fmt.Sprintf("%v", st)) } diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 724149c44..336bf5f70 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -35,7 +35,8 @@ type ControlServiceClient interface { TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) - // Forces a container removal to be signaled by the IR node with high probability. + // Forces a container removal to be signaled by the IR node with high + // probability. RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) } @@ -93,7 +94,8 @@ type ControlServiceServer interface { TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) - // Forces a container removal to be signaled by the IR node with high probability. + // Forces a container removal to be signaled by the IR node with high + // probability. RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error) } diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 828814b25..840e0be67 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -32,6 +32,8 @@ const ( HealthStatus_READY HealthStatus = 2 // IR application is shutting down. HealthStatus_SHUTTING_DOWN HealthStatus = 3 + // IR application is reconfiguring. + HealthStatus_RECONFIGURING HealthStatus = 4 ) // Enum value maps for HealthStatus. @@ -41,12 +43,14 @@ var ( 1: "STARTING", 2: "READY", 3: "SHUTTING_DOWN", + 4: "RECONFIGURING", } HealthStatus_value = map[string]int32{ "HEALTH_STATUS_UNDEFINED": 0, "STARTING": 1, "READY": 2, "SHUTTING_DOWN": 3, + "RECONFIGURING": 4, } ) @@ -144,17 +148,19 @@ var file_pkg_services_control_ir_types_proto_rawDesc = []byte{ 0x22, 0x36, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x69, 0x67, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2a, 0x57, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x03, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, - 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, + 0x4e, 0x47, 0x10, 0x04, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, + 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, + 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/pkg/services/control/ir/types.proto b/pkg/services/control/ir/types.proto index 9b6731cf8..901a55918 100644 --- a/pkg/services/control/ir/types.proto +++ b/pkg/services/control/ir/types.proto @@ -26,4 +26,7 @@ enum HealthStatus { // IR application is shutting down. SHUTTING_DOWN = 3; + + // IR application is reconfiguring. + RECONFIGURING = 4; } diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 4c293a4c0..63f96e11a 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.0 // source: pkg/services/tree/service.proto @@ -18,8 +18,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 const ( TreeService_Add_FullMethodName = "/tree.TreeService/Add" @@ -70,9 +70,8 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { } func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddResponse) - err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -80,9 +79,8 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp } func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -90,9 +88,8 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, } func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RemoveResponse) - err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -100,9 +97,8 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts } func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MoveResponse) - err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -110,9 +106,8 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g } func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -120,12 +115,11 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) if err != nil { return nil, err } - x := &treeServiceGetSubTreeClient{ClientStream: stream} + x := &treeServiceGetSubTreeClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -153,9 +147,8 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { } func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TreeListResponse) - err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -163,9 +156,8 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o } func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ApplyResponse) - err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -173,12 +165,11 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) if err != nil { return nil, err } - x := &treeServiceGetOpLogClient{ClientStream: stream} + x := &treeServiceGetOpLogClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -206,9 +197,8 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { } func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -383,7 +373,7 @@ func _TreeService_GetSubTree_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{ServerStream: stream}) + return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{stream}) } type TreeService_GetSubTreeServer interface { @@ -440,7 +430,7 @@ func _TreeService_GetOpLog_Handler(srv interface{}, stream grpc.ServerStream) er if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{ServerStream: stream}) + return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{stream}) } type TreeService_GetOpLogServer interface { From 149f8f4b08fcad89a9386c43aac18bd93e0a9c3e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 19 Aug 2024 15:50:35 +0300 Subject: [PATCH 19/21] [#1319] treeSvc: Do not wrap error from APE Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/ape.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 6e78bf4ec..ee4687911 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -138,7 +138,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, request, err := s.newAPERequest(ctx, namespace, cid, operation, role, publicKey) if err != nil { - return apeErr(err) + return fmt.Errorf("failed to create ape request: %w", err) } var cr engine.ChainRouter @@ -167,7 +167,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), groups) status, found, err := cr.IsAllowed(apechain.Ingress, rt, request) if err != nil { - return apeErr(err) + return err } if found && status == apechain.Allow { return nil From dd570c93445ff40865637c5c120a1f2e7acde332 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 19 Aug 2024 12:34:05 +0300 Subject: [PATCH 20/21] [#1318] metrics: Fix `container_size_bytes` for EC When node put chunk into EC container, `policer` may remove it as redundant. This chunk marked as removed. When parent object removed and `gc` start iterating over chunk, node count removing chunk twice. Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/metabase/inhume.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index c265fb217..b62accc43 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -236,7 +236,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes return err } } else if errors.As(err, &ecErr) { - err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value, targetKey) + err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value) if err != nil { return err } @@ -280,7 +280,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes, garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket, - ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, targetKey []byte, + ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, ) error { for _, chunk := range ecInfo.Chunks { chunkBuf := make([]byte, addressKeySize) @@ -296,11 +296,11 @@ func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *I if err != nil { return err } - err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, chunkObj, res) + chunkKey := addressKey(chunkAddr, chunkBuf) + err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, chunkKey, cnr, chunkObj, res) if err != nil { return err } - chunkKey := addressKey(chunkAddr, chunkBuf) if tomb != nil { _, err = db.markAsGC(graveyardBKT, garbageBKT, chunkKey) if err != nil { From 5aef303259e51fc5d586636a7213e17307905cbe Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 20 Aug 2024 16:18:46 +0300 Subject: [PATCH 21/21] [#1318] meta: Add test `TestInhumeECObject` Signed-off-by: Anton Nikiforov --- .../metabase/inhume_ec_test.go | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 pkg/local_object_storage/metabase/inhume_ec_test.go diff --git a/pkg/local_object_storage/metabase/inhume_ec_test.go b/pkg/local_object_storage/metabase/inhume_ec_test.go new file mode 100644 index 000000000..c3b1e72da --- /dev/null +++ b/pkg/local_object_storage/metabase/inhume_ec_test.go @@ -0,0 +1,116 @@ +package meta + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestInhumeECObject(t *testing.T) { + t.Parallel() + + db := New( + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{uint64(12)}), + ) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + ecChunk := oidtest.ID() + ecChunk2 := oidtest.ID() + ecParent := oidtest.ID() + tombstoneID := oidtest.ID() + + chunkObj := testutil.GenerateObjectWithCID(cnr) + chunkObj.SetContainerID(cnr) + chunkObj.SetID(ecChunk) + chunkObj.SetPayload([]byte{0, 1, 2, 3, 4}) + chunkObj.SetPayloadSize(uint64(5)) + chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0)) + + chunkObj2 := testutil.GenerateObjectWithCID(cnr) + chunkObj2.SetContainerID(cnr) + chunkObj2.SetID(ecChunk2) + chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + chunkObj2.SetPayloadSize(uint64(10)) + chunkObj2.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 1, 3, []byte{}, 0)) + + // put object with EC + + var prm PutPrm + prm.SetObject(chunkObj) + prm.SetStorageID([]byte("0/0")) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + + prm.SetObject(chunkObj2) + _, err = db.Put(context.Background(), prm) + require.NoError(t, err) + + var ecChunkAddress oid.Address + ecChunkAddress.SetContainer(cnr) + ecChunkAddress.SetObject(ecChunk) + + var ecParentAddress oid.Address + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ecParent) + + var chunkObjectAddress oid.Address + chunkObjectAddress.SetContainer(cnr) + chunkObjectAddress.SetObject(ecChunk) + + var getPrm GetPrm + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.NoError(t, err) + + var ecInfoError *objectSDK.ECInfoError + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &ecInfoError) + require.True(t, len(ecInfoError.ECInfo().Chunks) == 2 && + ecInfoError.ECInfo().Chunks[0].Index == 0 && + ecInfoError.ECInfo().Chunks[0].Total == 3) + + // inhume Chunk + var inhumePrm InhumePrm + var tombAddress oid.Address + inhumePrm.SetAddresses(chunkObjectAddress) + res, err := db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + require.True(t, len(res.deletionDetails) == 1) + require.True(t, res.deletionDetails[0].Size == 5) + + // inhume EC parent (like Delete does) + tombAddress.SetContainer(cnr) + tombAddress.SetObject(tombstoneID) + inhumePrm.SetAddresses(ecParentAddress) + inhumePrm.SetTombstoneAddress(tombAddress) + res, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + // Previously deleted chunk shouldn't be in the details, because it is marked as garbage + require.True(t, len(res.deletionDetails) == 1) + require.True(t, res.deletionDetails[0].Size == 10) + + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) +}