From c165d1a9b5ef6a3d1c28542acbba272cd0265f3f Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 28 Jun 2022 10:01:05 +0300 Subject: [PATCH] [#1556] Upgrade NeoFS SDK Go with changed container API Signed-off-by: Leonard Lyubich --- cmd/neofs-cli/internal/client/client.go | 2 +- cmd/neofs-cli/modules/container/create.go | 64 ++++------------ cmd/neofs-cli/modules/container/get.go | 55 ++++---------- cmd/neofs-node/container.go | 59 ++++++--------- go.mod | 4 +- go.sum | 8 +- pkg/core/container/fmt.go | 54 -------------- pkg/core/container/fmt_test.go | 73 ------------------- pkg/core/container/storage.go | 2 +- pkg/innerring/processors/audit/process.go | 11 +-- .../processors/container/process_container.go | 61 ++++------------ .../processors/container/process_eacl.go | 7 +- pkg/innerring/settlement.go | 14 +--- pkg/morph/client/container/get.go | 2 - pkg/morph/client/container/load.go | 14 +--- pkg/morph/client/container/put.go | 20 ++--- pkg/services/audit/task.go | 6 +- .../announcement/load/controller/calls.go | 8 +- .../load/controller/calls_test.go | 23 +++--- .../announcement/load/controller/deps.go | 18 ++--- .../announcement/load/controller/util.go | 2 +- .../announcement/load/route/calls.go | 14 +--- .../container/announcement/load/route/deps.go | 2 +- .../load/route/placement/calls.go | 8 +- .../container/announcement/load/route/util.go | 2 +- .../announcement/load/storage/storage.go | 18 ++--- .../announcement/load/storage/storage_test.go | 18 ++--- pkg/services/container/morph/executor.go | 18 ++++- pkg/services/container/morph/executor_test.go | 8 ++ pkg/services/object/acl/v2/classifier.go | 21 ++---- pkg/services/object/acl/v2/service.go | 4 +- pkg/services/object/get/get_test.go | 20 +++-- pkg/services/object/search/search_test.go | 16 ++-- .../object_manager/placement/traverser.go | 12 +-- .../placement/traverser_test.go | 9 ++- pkg/services/policer/check.go | 10 +-- 36 files changed, 207 insertions(+), 480 deletions(-) delete mode 100644 pkg/core/container/fmt.go delete mode 100644 pkg/core/container/fmt_test.go diff --git a/cmd/neofs-cli/internal/client/client.go b/cmd/neofs-cli/internal/client/client.go index 28a655fe2..bb46ec2ef 100644 --- a/cmd/neofs-cli/internal/client/client.go +++ b/cmd/neofs-cli/internal/client/client.go @@ -123,7 +123,7 @@ type GetContainerRes struct { } // Container returns structured of the requested container. -func (x GetContainerRes) Container() *container.Container { +func (x GetContainerRes) Container() container.Container { return x.cliRes.Container() } diff --git a/cmd/neofs-cli/modules/container/create.go b/cmd/neofs-cli/modules/container/create.go index f86862e7b..abb15d47c 100644 --- a/cmd/neofs-cli/modules/container/create.go +++ b/cmd/neofs-cli/modules/container/create.go @@ -4,11 +4,9 @@ import ( "errors" "fmt" "os" - "strconv" "strings" "time" - "github.com/google/uuid" internalclient "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/client" "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/common" "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/commonflags" @@ -19,13 +17,11 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/session" subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" "github.com/nspcc-dev/neofs-sdk-go/user" - versionSDK "github.com/nspcc-dev/neofs-sdk-go/version" "github.com/spf13/cobra" ) var ( containerACL string - containerNonce string containerPolicy string containerAttributes []string containerAwait bool @@ -52,18 +48,17 @@ It will be stored in sidechain when inner ring will accepts it.`, placementPolicy.RestrictSubnet(subnetID) } - attributes, err := parseAttributes(containerAttributes) + var cnr container.Container + cnr.Init() + + err = parseAttributes(&cnr, containerAttributes) common.ExitOnErr(cmd, "", err) var basicACL acl.Basic common.ExitOnErr(cmd, "decode basic ACL string: %w", basicACL.DecodeString(containerACL)) - nonce, err := parseNonce(containerNonce) - common.ExitOnErr(cmd, "", err) - key := key.GetOrGenerate(cmd) - cnr := container.New() var tok *session.Container sessionTokenPath, _ := cmd.Flags().GetString(commonflags.SessionToken) @@ -72,27 +67,22 @@ It will be stored in sidechain when inner ring will accepts it.`, common.ReadSessionToken(cmd, tok, sessionTokenPath) issuer := tok.Issuer() - cnr.SetOwnerID(&issuer) + cnr.SetOwner(issuer) } else { var idOwner user.ID user.IDFromKey(&idOwner, key.PublicKey) - cnr.SetOwnerID(&idOwner) + cnr.SetOwner(idOwner) } - ver := versionSDK.Current() - - cnr.SetVersion(&ver) - cnr.SetPlacementPolicy(placementPolicy) + cnr.SetPlacementPolicy(*placementPolicy) cnr.SetBasicACL(basicACL) - cnr.SetAttributes(attributes) - cnr.SetNonceUUID(nonce) cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC) var putPrm internalclient.PutContainerPrm putPrm.SetClient(cli) - putPrm.SetContainer(*cnr) + putPrm.SetContainer(cnr) if tok != nil { putPrm.WithinSession(*tok) @@ -137,7 +127,6 @@ func initContainerCreateCmd() { )) flags.StringVarP(&containerPolicy, "policy", "p", "", "QL-encoded or JSON-encoded placement policy or path to file with it") flags.StringSliceVarP(&containerAttributes, "attributes", "a", nil, "comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2") - flags.StringVarP(&containerNonce, "nonce", "n", "", "UUIDv4 nonce value for container") flags.BoolVar(&containerAwait, "await", false, "block execution until container is persisted") flags.StringVar(&containerName, "name", "", "container name attribute") flags.BoolVar(&containerNoTimestamp, "disable-timestamp", false, "disable timestamp container attribute") @@ -173,48 +162,23 @@ func parseContainerPolicy(policyString string) (*netmap.PlacementPolicy, error) return nil, errors.New("can't parse placement policy") } -func parseAttributes(attributes []string) ([]container.Attribute, error) { - result := make([]container.Attribute, len(attributes), len(attributes)+2) // name + timestamp attributes - +func parseAttributes(dst *container.Container, attributes []string) error { for i := range attributes { kvPair := strings.Split(attributes[i], attributeDelimiter) if len(kvPair) != 2 { - return nil, errors.New("invalid container attribute") + return errors.New("invalid container attribute") } - result[i].SetKey(kvPair[0]) - result[i].SetValue(kvPair[1]) + dst.SetAttribute(kvPair[0], kvPair[1]) } if !containerNoTimestamp { - index := len(result) - result = append(result, container.Attribute{}) - result[index].SetKey(container.AttributeTimestamp) - result[index].SetValue(strconv.FormatInt(time.Now().Unix(), 10)) + container.SetCreationTime(dst, time.Now()) } if containerName != "" { - index := len(result) - result = append(result, container.Attribute{}) - result[index].SetKey(container.AttributeName) - result[index].SetValue(containerName) + container.SetName(dst, containerName) } - return result, nil -} - -func parseNonce(nonce string) (uuid.UUID, error) { - if nonce == "" { - result := uuid.New() - common.PrintVerbose("Generating container nonce: %s", result) - - return result, nil - } - - uid, err := uuid.Parse(nonce) - if err != nil { - return uuid.UUID{}, fmt.Errorf("could not parse nonce: %w", err) - } - - return uid, nil + return nil } diff --git a/cmd/neofs-cli/modules/container/get.go b/cmd/neofs-cli/modules/container/get.go index 1bbc071a8..ccdb27277 100644 --- a/cmd/neofs-cli/modules/container/get.go +++ b/cmd/neofs-cli/modules/container/get.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key" "github.com/nspcc-dev/neofs-sdk-go/container" "github.com/nspcc-dev/neofs-sdk-go/container/acl" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/spf13/cobra" ) @@ -26,13 +27,12 @@ var getContainerInfoCmd = &cobra.Command{ Short: "Get container field info", Long: `Get container field info`, Run: func(cmd *cobra.Command, args []string) { - var cnr *container.Container + var cnr container.Container if containerPathFrom != "" { data, err := os.ReadFile(containerPathFrom) common.ExitOnErr(cmd, "can't read file: %w", err) - cnr = container.New() err = cnr.Unmarshal(data) common.ExitOnErr(cmd, "can't unmarshal container: %w", err) } else { @@ -62,8 +62,7 @@ var getContainerInfoCmd = &cobra.Command{ data, err = cnr.MarshalJSON() common.ExitOnErr(cmd, "can't JSON encode container: %w", err) } else { - data, err = cnr.Marshal() - common.ExitOnErr(cmd, "can't binary encode container: %w", err) + data = cnr.Marshal() } err = os.WriteFile(containerPathTo, data, 0644) @@ -90,11 +89,7 @@ func (x *stringWriter) WriteString(s string) (n int, err error) { return len(s), nil } -func prettyPrintContainer(cmd *cobra.Command, cnr *container.Container, jsonEncoding bool) { - if cnr == nil { - return - } - +func prettyPrintContainer(cmd *cobra.Command, cnr container.Container, jsonEncoding bool) { if jsonEncoding { data, err := cnr.MarshalJSON() if err != nil { @@ -111,45 +106,25 @@ func prettyPrintContainer(cmd *cobra.Command, cnr *container.Container, jsonEnco return } - id := container.CalculateID(cnr) + var id cid.ID + container.CalculateID(&id, cnr) cmd.Println("container ID:", id) - v := cnr.Version() - cmd.Printf("version: %d.%d\n", v.Major(), v.Minor()) - - cmd.Println("owner ID:", cnr.OwnerID()) + cmd.Println("owner ID:", cnr.Owner()) basicACL := cnr.BasicACL() prettyPrintBasicACL(cmd, basicACL) - for _, attribute := range cnr.Attributes() { - if attribute.Key() == container.AttributeTimestamp { - cmd.Printf("attribute: %s=%s (%s)\n", - attribute.Key(), - attribute.Value(), - common.PrettyPrintUnixTime(attribute.Value())) + cmd.Println("created:", container.CreatedAt(cnr)) - continue - } + cmd.Println("attributes:") + cnr.IterateAttributes(func(key, val string) { + cmd.Printf("\t%s=%s\n", key, val) + }) - cmd.Printf("attribute: %s=%s\n", attribute.Key(), attribute.Value()) - } - - nonce, err := cnr.NonceUUID() - if err == nil { - cmd.Println("nonce:", nonce) - } else { - cmd.Println("invalid nonce:", err) - } - - pp := cnr.PlacementPolicy() - if pp == nil { - cmd.Println("missing placement policy") - } else { - cmd.Println("placement policy:") - common.ExitOnErr(cmd, "write policy: %w", pp.WriteStringTo((*stringWriter)(cmd))) - cmd.Println() - } + cmd.Println("placement policy:") + common.ExitOnErr(cmd, "write policy: %w", cnr.PlacementPolicy().WriteStringTo((*stringWriter)(cmd))) + cmd.Println() } func prettyPrintBasicACL(cmd *cobra.Command, basicACL acl.Basic) { diff --git a/cmd/neofs-node/container.go b/cmd/neofs-node/container.go index 91780c48b..d00c2ee0f 100644 --- a/cmd/neofs-node/container.go +++ b/cmd/neofs-node/container.go @@ -213,16 +213,11 @@ type morphLoadWriter struct { key []byte } -func (w *morphLoadWriter) Put(a containerSDK.UsedSpaceAnnouncement) error { - cnr, ok := a.ContainerID() - if !ok { - return errors.New("missing container ID in load announcement") - } - +func (w *morphLoadWriter) Put(a containerSDK.SizeEstimation) error { w.log.Debug("save used space announcement in contract", zap.Uint64("epoch", a.Epoch()), - zap.Stringer("cid", cnr), - zap.Uint64("size", a.UsedSpace()), + zap.Stringer("cid", a.Container()), + zap.Uint64("size", a.Value()), ) prm := cntClient.AnnounceLoadPrm{} @@ -239,7 +234,7 @@ func (*morphLoadWriter) Close() error { type nopLoadWriter struct{} -func (nopLoadWriter) Put(containerSDK.UsedSpaceAnnouncement) error { +func (nopLoadWriter) Put(containerSDK.SizeEstimation) error { return nil } @@ -302,10 +297,10 @@ type remoteLoadAnnounceWriter struct { client client.Client - buf []containerSDK.UsedSpaceAnnouncement + buf []containerSDK.SizeEstimation } -func (r *remoteLoadAnnounceWriter) Put(a containerSDK.UsedSpaceAnnouncement) error { +func (r *remoteLoadAnnounceWriter) Put(a containerSDK.SizeEstimation) error { r.buf = append(r.buf, a) return nil @@ -354,11 +349,6 @@ func (l *loadPlacementBuilder) buildPlacement(epoch uint64, idCnr cid.ID) ([][]n return nil, nil, err } - policy := cnr.Value.PlacementPolicy() - if policy == nil { - return nil, nil, errors.New("missing placement policy in container") - } - nm, err := l.nmSrc.GetNetMapByEpoch(epoch) if err != nil { return nil, nil, fmt.Errorf("could not get network map: %w", err) @@ -367,7 +357,7 @@ func (l *loadPlacementBuilder) buildPlacement(epoch uint64, idCnr cid.ID) ([][]n binCnr := make([]byte, sha256.Size) idCnr.Encode(binCnr) - cnrNodes, err := nm.ContainerNodes(*policy, binCnr) + cnrNodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), binCnr) if err != nil { return nil, nil, fmt.Errorf("could not build container nodes: %w", err) } @@ -403,15 +393,15 @@ func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontr zap.Stringer("cid", idList[i]), ) - a := containerSDK.NewAnnouncement() - a.SetContainerID(idList[i]) - a.SetUsedSpace(sz) + var a containerSDK.SizeEstimation + a.SetContainer(idList[i]) + a.SetValue(sz) - if f != nil && !f(*a) { + if f != nil && !f(a) { continue } - if err := h(*a); err != nil { + if err := h(a); err != nil { return err } } @@ -479,8 +469,15 @@ func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *container return nil, fmt.Errorf("could not initialize container's used space writer: %w", err) } + var est containerSDK.SizeEstimation + for _, aV2 := range req.GetBody().GetAnnouncements() { - if err := c.processLoadValue(ctx, *containerSDK.NewAnnouncementFromV2(&aV2), passedRoute, w); err != nil { + err = est.ReadFromV2(aV2) + if err != nil { + return nil, fmt.Errorf("invalid size announcement: %w", err) + } + + if err := c.processLoadValue(ctx, est, passedRoute, w); err != nil { return nil, err } } @@ -527,14 +524,9 @@ func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, return false, nil } -func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.UsedSpaceAnnouncement, +func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.SizeEstimation, route []loadroute.ServerInfo, w loadcontroller.Writer) error { - cnr, ok := a.ContainerID() - if !ok { - return errors.New("missing container ID in load announcement") - } - - fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), cnr, route[0].PublicKey()) + fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), a.Container(), route[0].PublicKey()) if err != nil { return fmt.Errorf("could not verify that the sender belongs to the container: %w", err) } else if !fromCnr { @@ -591,13 +583,8 @@ func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) return nil, err } - idOwner := cnr.Value.OwnerID() - if idOwner == nil { - return nil, errors.New("missing container owner") - } - if m.cacheEnabled { - m.lists.InvalidateContainerList(*idOwner) + m.lists.InvalidateContainerList(cnr.Value.Owner()) } return containerID, nil diff --git a/go.mod b/go.mod index 051d0d336..af4c03af7 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,9 @@ require ( github.com/nspcc-dev/hrw v1.0.9 github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220609082921-2c9fb2044242 github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220601120906-3bec6657f5c5 // indirect - github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220621170933-dd233c3fbc84 + github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220630100506-c6f7ab3ef1bf github.com/nspcc-dev/neofs-contract v0.15.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220623080532-aa5ee1dcde1c + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220704082116-2ad89085a341 github.com/nspcc-dev/tzhash v1.5.2 github.com/panjf2000/ants/v2 v2.4.0 github.com/paulmach/orb v0.2.2 diff --git a/go.sum b/go.sum index 0877a207d..c539ac658 100644 --- a/go.sum +++ b/go.sum @@ -399,8 +399,8 @@ github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220601120906-3bec6657f5c5 h1:bW github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220601120906-3bec6657f5c5/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220621170933-dd233c3fbc84 h1:4tZSQ2DL/oatbte35+vDSt2nYpQ0G2et1DrpxodGwRM= -github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220621170933-dd233c3fbc84/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220630100506-c6f7ab3ef1bf h1:QRPx+DdyN2KmJ5/oDYH4c86Bl81d1ZacQL5Q9IC9wZA= +github.com/nspcc-dev/neofs-api-go/v2 v2.12.3-0.20220630100506-c6f7ab3ef1bf/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= github.com/nspcc-dev/neofs-contract v0.15.1 h1:1r27t4SGKF7W1PRPOIfircEXHvALThNYNagT+SIabcA= github.com/nspcc-dev/neofs-contract v0.15.1/go.mod h1:kxO5ZTqdzFnRM5RMvM+Fhd+3GGrJo6AmG2ZyA9OCqqQ= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -409,8 +409,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnB github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220623080532-aa5ee1dcde1c h1:1kYefIk0WdUZJPsZ5HXN7hdNgsb3gEP+/AVMS6lxUzI= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220623080532-aa5ee1dcde1c/go.mod h1:EpzpilARa1/7Pgtn8qB/iXXyvC1AIzhlm8mbU+S52MU= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220704082116-2ad89085a341 h1:poXrrTjCClTMUpEZ7xlQ3Pk4+vtLH+EqaTduBmn+Z9Y= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220704082116-2ad89085a341/go.mod h1:ck/wjPGFZ7mqcz6vZLMuQFEfL1Qu5zVrBrBJrH0VjOo= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/pkg/core/container/fmt.go b/pkg/core/container/fmt.go deleted file mode 100644 index 4a133a536..000000000 --- a/pkg/core/container/fmt.go +++ /dev/null @@ -1,54 +0,0 @@ -package container - -import ( - "errors" - "fmt" - - "github.com/nspcc-dev/neofs-node/pkg/core/version" - "github.com/nspcc-dev/neofs-sdk-go/container" -) - -var ( - errNilPolicy = errors.New("placement policy is nil") - errRepeatedAttributes = errors.New("repeated attributes found") - errEmptyAttribute = errors.New("empty attribute found") -) - -// CheckFormat conducts an initial check of the v2 container data. -// -// It is expected that if a container fails this test, -// it will not be approved by the inner ring. -func CheckFormat(c *container.Container) error { - if c.PlacementPolicy() == nil { - return errNilPolicy - } - - if v := c.Version(); v == nil || !version.IsValid(*v) { - return fmt.Errorf("incorrect version %s", v) - } - - if c.OwnerID() == nil { - return errors.New("missing owner") - } - - if _, err := c.NonceUUID(); err != nil { - return fmt.Errorf("incorrect nonce: %w", err) - } - - // check if there are repeated attributes - attrs := c.Attributes() - uniqueAttr := make(map[string]struct{}, len(attrs)) - for _, attr := range attrs { - if _, exists := uniqueAttr[attr.Key()]; exists { - return errRepeatedAttributes - } - - if attr.Value() == "" { - return errEmptyAttribute - } - - uniqueAttr[attr.Key()] = struct{}{} - } - - return nil -} diff --git a/pkg/core/container/fmt_test.go b/pkg/core/container/fmt_test.go deleted file mode 100644 index 65fc3d937..000000000 --- a/pkg/core/container/fmt_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package container - -import ( - "testing" - - "github.com/google/uuid" - "github.com/nspcc-dev/neofs-node/pkg/util/test" - "github.com/nspcc-dev/neofs-sdk-go/container" - netmaptest "github.com/nspcc-dev/neofs-sdk-go/netmap/test" - "github.com/nspcc-dev/neofs-sdk-go/user" - "github.com/nspcc-dev/neofs-sdk-go/version" - "github.com/stretchr/testify/require" -) - -func TestCheckFormat(t *testing.T) { - c := container.New() - - require.Error(t, CheckFormat(c)) - - policy := netmaptest.PlacementPolicy() - c.SetPlacementPolicy(&policy) - - require.Error(t, CheckFormat(c)) - - ver := version.Current() - c.SetVersion(&ver) - - require.Error(t, CheckFormat(c)) - - var idUser user.ID - user.IDFromKey(&idUser, test.DecodeKey(-1).PublicKey) - - c.SetOwnerID(&idUser) - - // set incorrect nonce - cV2 := c.ToV2() - cV2.SetNonce([]byte{1, 2, 3}) - c = container.NewContainerFromV2(cV2) - - require.Error(t, CheckFormat(c)) - - c.SetNonceUUID(uuid.New()) - - require.NoError(t, CheckFormat(c)) - - // set empty value attribute - var attr1 container.Attribute - attr1.SetKey("attr") - attrs := container.Attributes{attr1} - - c.SetAttributes(attrs) - - require.ErrorIs(t, CheckFormat(c), errEmptyAttribute) - - // add same key attribute - var attr2 container.Attribute - attr2.SetKey(attr1.Key()) - attr2.SetValue("val") - - attrs[0].SetValue(attr2.Value()) - - attrs = append(attrs, attr2) - - c.SetAttributes(attrs) - - require.ErrorIs(t, CheckFormat(c), errRepeatedAttributes) - - attrs[1].SetKey(attr1.Key() + "smth") - - c.SetAttributes(attrs) - - require.NoError(t, CheckFormat(c)) -} diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index 8ca339866..7babe9bf9 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -14,7 +14,7 @@ import ( // Container groups information about the NeoFS container stored in the NeoFS network. type Container struct { // Container structure. - Value *container.Container + Value container.Container // Signature of the Value. Signature neofscrypto.Signature diff --git a/pkg/innerring/processors/audit/process.go b/pkg/innerring/processors/audit/process.go index 36849b619..058fbb5e9 100644 --- a/pkg/innerring/processors/audit/process.go +++ b/pkg/innerring/processors/audit/process.go @@ -60,19 +60,10 @@ func (ap *Processor) processStartAudit(epoch uint64) { continue } - policy := cnr.Value.PlacementPolicy() - if policy == nil { - log.Error("missing placement policy in container, ignore", - zap.Stringer("cid", containers[i]), - ) - - continue - } - containers[i].Encode(pivot) // find all container nodes for current epoch - nodes, err := nm.ContainerNodes(*policy, pivot) + nodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), pivot) if err != nil { log.Info("can't build placement for container, ignore", zap.Stringer("cid", containers[i]), diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 081d6e8b8..ba0930608 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -1,11 +1,9 @@ package container import ( - "errors" "fmt" "github.com/nspcc-dev/neo-go/pkg/network/payload" - "github.com/nspcc-dev/neofs-node/pkg/core/container" cntClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/container" morphsubnet "github.com/nspcc-dev/neofs-node/pkg/morph/client/subnet" "github.com/nspcc-dev/neofs-node/pkg/morph/event" @@ -30,7 +28,7 @@ type putEvent interface { type putContainerContext struct { e putEvent - name, zone string // from container structure + d containerSDK.Domain } // Process a new container from the user by checking the container sanity @@ -59,21 +57,15 @@ func (cp *Processor) processContainerPut(put putEvent) { func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { binCnr := ctx.e.Container() - - cnr := containerSDK.New() + var cnr containerSDK.Container err := cnr.Unmarshal(binCnr) if err != nil { return fmt.Errorf("invalid binary container: %w", err) } - ownerContainer := cnr.OwnerID() - if ownerContainer == nil { - return errors.New("missing container owner") - } - err = cp.verifySignature(signatureVerificationData{ - ownerContainer: *ownerContainer, + ownerContainer: cnr.Owner(), verb: session.VerbContainerPut, binTokenSession: ctx.e.SessionToken(), binPublicKey: ctx.e.PublicKey(), @@ -96,12 +88,6 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return fmt.Errorf("NNS: %w", err) } - // perform format check - err = container.CheckFormat(cnr) - if err != nil { - return fmt.Errorf("incorrect container format: %w", err) - } - return nil } @@ -116,8 +102,8 @@ func (cp *Processor) approvePutContainer(ctx *putContainerContext) { prm.SetKey(e.PublicKey()) prm.SetSignature(e.Signature()) prm.SetToken(e.SessionToken()) - prm.SetName(ctx.name) - prm.SetZone(ctx.zone) + prm.SetName(ctx.d.Name()) + prm.SetZone(ctx.d.Zone()) if nr := e.NotaryRequest(); nr != nil { // put event was received via Notary service @@ -169,13 +155,8 @@ func (cp *Processor) checkDeleteContainer(e *containerEvent.Delete) error { return fmt.Errorf("could not receive the container: %w", err) } - ownerContainer := cnr.Value.OwnerID() - if ownerContainer == nil { - return errors.New("missing container owner") - } - err = cp.verifySignature(signatureVerificationData{ - ownerContainer: *ownerContainer, + ownerContainer: cnr.Value.Owner(), verb: session.VerbContainerDelete, idContainerSet: true, idContainer: idCnr, @@ -213,47 +194,37 @@ func (cp *Processor) approveDeleteContainer(e *containerEvent.Delete) { } } -func checkNNS(ctx *putContainerContext, cnr *containerSDK.Container) error { - // fetch native name and zone - ctx.name, ctx.zone = containerSDK.GetNativeNameWithZone(cnr) +func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { + // fetch domain info + ctx.d = containerSDK.ReadDomain(cnr) // if PutNamed event => check if values in container correspond to args if named, ok := ctx.e.(interface { Name() string Zone() string }); ok { - if name := named.Name(); name != ctx.name { - return fmt.Errorf("names differ %s/%s", name, ctx.name) + if name := named.Name(); name != ctx.d.Name() { + return fmt.Errorf("names differ %s/%s", name, ctx.d.Name()) } - if zone := named.Zone(); zone != ctx.zone { - return fmt.Errorf("zones differ %s/%s", zone, ctx.zone) + if zone := named.Zone(); zone != ctx.d.Zone() { + return fmt.Errorf("zones differ %s/%s", zone, ctx.d.Zone()) } } return nil } -func checkSubnet(subCli *morphsubnet.Client, cnr *containerSDK.Container) error { - owner := cnr.OwnerID() - if owner == nil { - return errors.New("missing owner") - } - - policy := cnr.PlacementPolicy() - if policy == nil { - return errors.New("missing placement policy") - } - +func checkSubnet(subCli *morphsubnet.Client, cnr containerSDK.Container) error { prm := morphsubnet.UserAllowedPrm{} - subID := policy.Subnet() + subID := cnr.PlacementPolicy().Subnet() if subnetid.IsZero(subID) { return nil } prm.SetID(subID.Marshal()) - prm.SetClient(owner.WalletBytes()) + prm.SetClient(cnr.Owner().WalletBytes()) res, err := subCli.UserAllowed(prm) if err != nil { diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index b8c9e4d19..db862aeeb 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -56,13 +56,8 @@ func (cp *Processor) checkSetEACL(e container.SetEACL) error { return errors.New("ACL extension disabled by container basic ACL") } - ownerContainer := cnr.Value.OwnerID() - if ownerContainer == nil { - return errors.New("missing container owner") - } - err = cp.verifySignature(signatureVerificationData{ - ownerContainer: *ownerContainer, + ownerContainer: cnr.Value.Owner(), verb: session.VerbContainerSetEACL, idContainerSet: true, idContainer: idCnr, diff --git a/pkg/innerring/settlement.go b/pkg/innerring/settlement.go index 349f1b5b5..8555fcb28 100644 --- a/pkg/innerring/settlement.go +++ b/pkg/innerring/settlement.go @@ -6,7 +6,6 @@ import ( "crypto/elliptic" "crypto/sha256" "encoding/hex" - "errors" "fmt" "math/big" @@ -93,8 +92,8 @@ func (n nodeInfoWrapper) Price() *big.Int { return big.NewInt(int64(n.ni.Price())) } -func (c *containerWrapper) Owner() user.ID { - return *(*containerAPI.Container)(c).OwnerID() +func (c containerWrapper) Owner() user.ID { + return (containerAPI.Container)(c).Owner() } func (s settlementDeps) AuditResultsForEpoch(epoch uint64) ([]*auditAPI.Result, error) { @@ -123,7 +122,7 @@ func (s settlementDeps) ContainerInfo(cid cid.ID) (common.ContainerInfo, error) return nil, fmt.Errorf("could not get container from storage: %w", err) } - return (*containerWrapper)(cnr.Value), nil + return (containerWrapper)(cnr.Value), nil } func (s settlementDeps) buildContainer(e uint64, cid cid.ID) ([][]netmapAPI.NodeInfo, *netmapAPI.NetMap, error) { @@ -147,16 +146,11 @@ func (s settlementDeps) buildContainer(e uint64, cid cid.ID) ([][]netmapAPI.Node return nil, nil, fmt.Errorf("could not get container from sidechain: %w", err) } - policy := cnr.Value.PlacementPolicy() - if policy == nil { - return nil, nil, errors.New("missing placement policy in container") - } - binCnr := make([]byte, sha256.Size) cid.Encode(binCnr) cn, err := nm.ContainerNodes( - *policy, + cnr.Value.PlacementPolicy(), binCnr, // may be replace pivot calculation to neofs-api-go ) if err != nil { diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index adaef4e49..3ebb57a55 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -11,7 +11,6 @@ import ( core "github.com/nspcc-dev/neofs-node/pkg/core/container" "github.com/nspcc-dev/neofs-node/pkg/morph/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" - "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto" "github.com/nspcc-dev/neofs-sdk-go/session" @@ -90,7 +89,6 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { var cnr containercore.Container - cnr.Value = container.New() if err := cnr.Value.Unmarshal(cnrBytes); err != nil { // use other major version if there any return nil, fmt.Errorf("can't unmarshal container: %w", err) diff --git a/pkg/morph/client/container/load.go b/pkg/morph/client/container/load.go index 6f9dca47a..7a4dd37e4 100644 --- a/pkg/morph/client/container/load.go +++ b/pkg/morph/client/container/load.go @@ -2,7 +2,6 @@ package container import ( "crypto/sha256" - "errors" "fmt" v2refs "github.com/nspcc-dev/neofs-api-go/v2/refs" @@ -13,14 +12,14 @@ import ( // AnnounceLoadPrm groups parameters of AnnounceLoad operation. type AnnounceLoadPrm struct { - a container.UsedSpaceAnnouncement + a container.SizeEstimation key []byte client.InvokePrmOptional } // SetAnnouncement sets announcement. -func (a2 *AnnounceLoadPrm) SetAnnouncement(a container.UsedSpaceAnnouncement) { +func (a2 *AnnounceLoadPrm) SetAnnouncement(a container.SizeEstimation) { a2.a = a } @@ -34,17 +33,12 @@ func (a2 *AnnounceLoadPrm) SetReporter(key []byte) { // // Returns any error encountered that caused the saving to interrupt. func (c *Client) AnnounceLoad(p AnnounceLoadPrm) error { - cnr, ok := p.a.ContainerID() - if !ok { - return errors.New("missing container for load announcement") - } - binCnr := make([]byte, sha256.Size) - cnr.Encode(binCnr) + p.a.Container().Encode(binCnr) prm := client.InvokePrm{} prm.SetMethod(putSizeMethod) - prm.SetArgs(p.a.Epoch(), binCnr, p.a.UsedSpace(), p.key) + prm.SetArgs(p.a.Epoch(), binCnr, p.a.Value(), p.key) prm.InvokePrmOptional = p.InvokePrmOptional err := c.client.Invoke(prm) diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 39ecf62d1..518de644f 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -1,7 +1,6 @@ package container import ( - "crypto/sha256" "fmt" "github.com/nspcc-dev/neofs-api-go/v2/refs" @@ -16,21 +15,14 @@ import ( // // Returns error if container is nil. func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { - if cnr.Value == nil { - return nil, errNilArgument - } + data := cnr.Value.Marshal() - data, err := cnr.Value.Marshal() - if err != nil { - return nil, fmt.Errorf("can't marshal container: %w", err) - } - - name, zone := container.GetNativeNameWithZone(cnr.Value) + d := container.ReadDomain(cnr.Value) var prm PutPrm prm.SetContainer(data) - prm.SetName(name) - prm.SetZone(zone) + prm.SetName(d.Name()) + prm.SetZone(d.Zone()) if cnr.Session != nil { prm.SetToken(cnr.Session.Marshal()) @@ -43,13 +35,13 @@ func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { prm.SetKey(sigV2.GetKey()) prm.SetSignature(sigV2.GetSign()) - err = c.Put(prm) + err := c.Put(prm) if err != nil { return nil, err } var id cid.ID - id.SetSHA256(sha256.Sum256(data)) + container.CalculateIDFromBinary(&id, data) return &id, nil } diff --git a/pkg/services/audit/task.go b/pkg/services/audit/task.go index 737a0a12c..f207033a0 100644 --- a/pkg/services/audit/task.go +++ b/pkg/services/audit/task.go @@ -17,7 +17,7 @@ type Task struct { idCnr cid.ID - cnr *container.Container + cnr container.Container nm *netmap.NetMap @@ -69,7 +69,7 @@ func (t *Task) ContainerID() cid.ID { } // WithContainerStructure sets structure of the container under audit. -func (t *Task) WithContainerStructure(cnr *container.Container) *Task { +func (t *Task) WithContainerStructure(cnr container.Container) *Task { if t != nil { t.cnr = cnr } @@ -78,7 +78,7 @@ func (t *Task) WithContainerStructure(cnr *container.Container) *Task { } // ContainerStructure returns structure of the container under audit. -func (t *Task) ContainerStructure() *container.Container { +func (t *Task) ContainerStructure() container.Container { return t.cnr } diff --git a/pkg/services/container/announcement/load/controller/calls.go b/pkg/services/container/announcement/load/controller/calls.go index 24b5bace6..b209ef74c 100644 --- a/pkg/services/container/announcement/load/controller/calls.go +++ b/pkg/services/container/announcement/load/controller/calls.go @@ -29,7 +29,7 @@ type announceContext struct { commonContext } -// Start starts the processing of UsedSpaceAnnouncement values. +// Start starts the processing of container.SizeEstimation values. // // Single Start operation overtakes all data from LocalMetrics to // LocalAnnouncementTarget (Controller's parameters). @@ -82,10 +82,10 @@ func (c *announceContext) announce() { // iterate over all collected metrics and write them to the target err = metricsIterator.Iterate( - func(container.UsedSpaceAnnouncement) bool { + func(container.SizeEstimation) bool { return true // local metrics don't know about epochs }, - func(a container.UsedSpaceAnnouncement) error { + func(a container.SizeEstimation) error { a.SetEpoch(c.epoch) // set epoch explicitly return targetWriter.Put(a) }, @@ -179,7 +179,7 @@ type stopContext struct { commonContext } -// Stop interrupts the processing of UsedSpaceAnnouncement values. +// Stop interrupts the processing of container.SizeEstimation values. // // Single Stop operation releases an announcement context and overtakes // all data from AnnouncementAccumulator to ResultReceiver (Controller's diff --git a/pkg/services/container/announcement/load/controller/calls_test.go b/pkg/services/container/announcement/load/controller/calls_test.go index 199ca57ce..e61c9fe78 100644 --- a/pkg/services/container/announcement/load/controller/calls_test.go +++ b/pkg/services/container/announcement/load/controller/calls_test.go @@ -19,12 +19,12 @@ type testAnnouncementStorage struct { mtx sync.RWMutex - m map[uint64][]container.UsedSpaceAnnouncement + m map[uint64][]container.SizeEstimation } func newTestStorage() *testAnnouncementStorage { return &testAnnouncementStorage{ - m: make(map[uint64][]container.UsedSpaceAnnouncement), + m: make(map[uint64][]container.SizeEstimation), } } @@ -61,7 +61,7 @@ func (s *testAnnouncementStorage) InitWriter(context.Context) (loadcontroller.Wr return s, nil } -func (s *testAnnouncementStorage) Put(v container.UsedSpaceAnnouncement) error { +func (s *testAnnouncementStorage) Put(v container.SizeEstimation) error { s.mtx.Lock() s.m[v.Epoch()] = append(s.m[v.Epoch()], v) s.mtx.Unlock() @@ -73,12 +73,11 @@ func (s *testAnnouncementStorage) Close() error { return nil } -func randAnnouncement() container.UsedSpaceAnnouncement { - a := container.NewAnnouncement() - a.SetContainerID(cidtest.ID()) - a.SetUsedSpace(rand.Uint64()) +func randAnnouncement() (a container.SizeEstimation) { + a.SetContainer(cidtest.ID()) + a.SetValue(rand.Uint64()) - return *a + return } func TestSimpleScenario(t *testing.T) { @@ -116,7 +115,7 @@ func TestSimpleScenario(t *testing.T) { const goodNum = 4 // create 2 random values for processing epoch and 1 for some different - announces := make([]container.UsedSpaceAnnouncement, 0, goodNum) + announces := make([]container.SizeEstimation, 0, goodNum) for i := 0; i < goodNum; i++ { a := randAnnouncement() @@ -174,13 +173,13 @@ func TestSimpleScenario(t *testing.T) { wg.Wait() // result target should contain all "good" announcements and shoult not container the "bad" one - var res []container.UsedSpaceAnnouncement + var res []container.SizeEstimation err := resultStorage.Iterate( - func(a container.UsedSpaceAnnouncement) bool { + func(a container.SizeEstimation) bool { return true }, - func(a container.UsedSpaceAnnouncement) error { + func(a container.SizeEstimation) error { res = append(res, a) return nil }, diff --git a/pkg/services/container/announcement/load/controller/deps.go b/pkg/services/container/announcement/load/controller/deps.go index fb199a3c0..83bd8836d 100644 --- a/pkg/services/container/announcement/load/controller/deps.go +++ b/pkg/services/container/announcement/load/controller/deps.go @@ -7,22 +7,22 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/container" ) -// UsedSpaceHandler describes the signature of the UsedSpaceAnnouncement +// UsedSpaceHandler describes the signature of the container.SizeEstimation // value handling function. // // Termination of processing without failures is usually signaled // with a zero error, while a specific value may describe the reason // for failure. -type UsedSpaceHandler func(container.UsedSpaceAnnouncement) error +type UsedSpaceHandler func(container.SizeEstimation) error // UsedSpaceFilter describes the signature of the function for // checking whether a value meets a certain criterion. // // Return of true means conformity, false - vice versa. -type UsedSpaceFilter func(container.UsedSpaceAnnouncement) bool +type UsedSpaceFilter func(container.SizeEstimation) bool // Iterator is a group of methods provided by entity -// which can iterate over a group of UsedSpaceAnnouncement values. +// which can iterate over a group of container.SizeEstimation values. type Iterator interface { // Iterate must start an iterator over values that // meet the filter criterion (returns true). @@ -37,7 +37,7 @@ type Iterator interface { // IteratorProvider is a group of methods provided // by entity which generates iterators over -// UsedSpaceAnnouncement values. +// container.SizeEstimation values. type IteratorProvider interface { // InitIterator should return an initialized Iterator. // @@ -49,12 +49,12 @@ type IteratorProvider interface { InitIterator(context.Context) (Iterator, error) } -// Writer describes the interface for storing UsedSpaceAnnouncement values. +// Writer describes the interface for storing container.SizeEstimation values. // // This interface is provided by both local storage // of values and remote (wrappers over the RPC). type Writer interface { - // Put performs a write operation of UsedSpaceAnnouncement value + // Put performs a write operation of container.SizeEstimation value // and returns any error encountered. // // All values after the Close call must be flushed to the @@ -62,7 +62,7 @@ type Writer interface { // Close operation. // // Put must not be called after Close. - Put(container.UsedSpaceAnnouncement) error + Put(container.SizeEstimation) error // Close exits with method-providing Writer. // @@ -75,7 +75,7 @@ type Writer interface { // WriterProvider is a group of methods provided // by entity which generates keepers of -// UsedSpaceAnnouncement values. +// container.SizeEstimation values. type WriterProvider interface { // InitWriter should return an initialized Writer. // diff --git a/pkg/services/container/announcement/load/controller/util.go b/pkg/services/container/announcement/load/controller/util.go index 83b0deaf2..3542c0492 100644 --- a/pkg/services/container/announcement/load/controller/util.go +++ b/pkg/services/container/announcement/load/controller/util.go @@ -7,7 +7,7 @@ import ( ) func usedSpaceFilterEpochEQ(epoch uint64) UsedSpaceFilter { - return func(a container.UsedSpaceAnnouncement) bool { + return func(a container.SizeEstimation) bool { return a.Epoch() == epoch } } diff --git a/pkg/services/container/announcement/load/route/calls.go b/pkg/services/container/announcement/load/route/calls.go index 5f675c62a..f18c10e83 100644 --- a/pkg/services/container/announcement/load/route/calls.go +++ b/pkg/services/container/announcement/load/route/calls.go @@ -3,7 +3,6 @@ package loadroute import ( "context" "encoding/hex" - "errors" "sync" loadcontroller "github.com/nspcc-dev/neofs-node/pkg/services/container/announcement/load/controller" @@ -71,7 +70,7 @@ type routeKey struct { type valuesRoute struct { route []ServerInfo - values []container.UsedSpaceAnnouncement + values []container.SizeEstimation } type loadWriter struct { @@ -85,18 +84,13 @@ type loadWriter struct { mServers map[string]loadcontroller.Writer } -func (w *loadWriter) Put(a container.UsedSpaceAnnouncement) error { - cnr, ok := a.ContainerID() - if !ok { - return errors.New("missing container in load announcement") - } - +func (w *loadWriter) Put(a container.SizeEstimation) error { w.routeMtx.Lock() defer w.routeMtx.Unlock() key := routeKey{ epoch: a.Epoch(), - cid: cnr.EncodeToString(), + cid: a.Container().EncodeToString(), } routeValues, ok := w.mRoute[key] @@ -110,7 +104,7 @@ func (w *loadWriter) Put(a container.UsedSpaceAnnouncement) error { routeValues = &valuesRoute{ route: route, - values: []container.UsedSpaceAnnouncement{a}, + values: []container.SizeEstimation{a}, } w.mRoute[key] = routeValues diff --git a/pkg/services/container/announcement/load/route/deps.go b/pkg/services/container/announcement/load/route/deps.go index 77b182ce3..8a27d7956 100644 --- a/pkg/services/container/announcement/load/route/deps.go +++ b/pkg/services/container/announcement/load/route/deps.go @@ -33,7 +33,7 @@ type Builder interface { // in that list (means that point is the last point in one of the route groups), // returned route must contain nil point that should be interpreted as signal to, // among sending to other route points, save the announcement in that point. - NextStage(a container.UsedSpaceAnnouncement, passed []ServerInfo) ([]ServerInfo, error) + NextStage(a container.SizeEstimation, passed []ServerInfo) ([]ServerInfo, error) } // RemoteWriterProvider describes the component diff --git a/pkg/services/container/announcement/load/route/placement/calls.go b/pkg/services/container/announcement/load/route/placement/calls.go index 2e81a32aa..0c4fa702d 100644 --- a/pkg/services/container/announcement/load/route/placement/calls.go +++ b/pkg/services/container/announcement/load/route/placement/calls.go @@ -2,7 +2,6 @@ package placementrouter import ( "bytes" - "errors" "fmt" netmapcore "github.com/nspcc-dev/neofs-node/pkg/core/netmap" @@ -16,15 +15,12 @@ import ( // If passed route has more than one point, then endpoint of the route is reached. // // The traversed route is not checked, it is assumed to be correct. -func (b *Builder) NextStage(a container.UsedSpaceAnnouncement, passed []loadroute.ServerInfo) ([]loadroute.ServerInfo, error) { +func (b *Builder) NextStage(a container.SizeEstimation, passed []loadroute.ServerInfo) ([]loadroute.ServerInfo, error) { if len(passed) > 1 { return nil, nil } - cnr, ok := a.ContainerID() - if !ok { - return nil, errors.New("missing container in load announcement") - } + cnr := a.Container() placement, err := b.placementBuilder.BuildPlacement(a.Epoch(), cnr) if err != nil { diff --git a/pkg/services/container/announcement/load/route/util.go b/pkg/services/container/announcement/load/route/util.go index 950bcbc1d..b47934735 100644 --- a/pkg/services/container/announcement/load/route/util.go +++ b/pkg/services/container/announcement/load/route/util.go @@ -12,7 +12,7 @@ var errWrongRoute = errors.New("wrong route") // CheckRoute checks if the route is a route correctly constructed by the builder for value a. // // Returns nil if route is correct, otherwise an error clarifying the inconsistency. -func CheckRoute(builder Builder, a container.UsedSpaceAnnouncement, route []ServerInfo) error { +func CheckRoute(builder Builder, a container.SizeEstimation, route []ServerInfo) error { for i := 1; i < len(route); i++ { servers, err := builder.NextStage(a, route[:i]) if err != nil { diff --git a/pkg/services/container/announcement/load/storage/storage.go b/pkg/services/container/announcement/load/storage/storage.go index 92bcf28b9..9957248e6 100644 --- a/pkg/services/container/announcement/load/storage/storage.go +++ b/pkg/services/container/announcement/load/storage/storage.go @@ -1,7 +1,6 @@ package loadstorage import ( - "errors" "sort" "sync" @@ -10,7 +9,7 @@ import ( ) type usedSpaceEstimations struct { - announcement container.UsedSpaceAnnouncement + announcement container.SizeEstimation sizes []uint64 } @@ -22,7 +21,7 @@ type storageKey struct { } // Storage represents in-memory storage of -// UsedSpaceAnnouncement values. +// container.SizeEstimation values. // // The write operation has the usual behavior - to save // the next number of used container space for a specific epoch. @@ -63,18 +62,13 @@ func New(_ Prm) *Storage { // to the list of already saved values. // // Always returns nil error. -func (s *Storage) Put(a container.UsedSpaceAnnouncement) error { - cnr, ok := a.ContainerID() - if !ok { - return errors.New("missing container in load announcement") - } - +func (s *Storage) Put(a container.SizeEstimation) error { s.mtx.Lock() { key := storageKey{ epoch: a.Epoch(), - cid: cnr.EncodeToString(), + cid: a.Container().EncodeToString(), } estimations, ok := s.mItems[key] @@ -87,7 +81,7 @@ func (s *Storage) Put(a container.UsedSpaceAnnouncement) error { s.mItems[key] = estimations } - estimations.sizes = append(estimations.sizes, a.UsedSpace()) + estimations.sizes = append(estimations.sizes, a.Value()) } s.mtx.Unlock() @@ -110,7 +104,7 @@ func (s *Storage) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.Use for _, v := range s.mItems { if f(v.announcement) { // calculate estimation based on 90th percentile - v.announcement.SetUsedSpace(finalEstimation(v.sizes)) + v.announcement.SetValue(finalEstimation(v.sizes)) if err = h(v.announcement); err != nil { break diff --git a/pkg/services/container/announcement/load/storage/storage_test.go b/pkg/services/container/announcement/load/storage/storage_test.go index 6ca13466a..4b1881b8d 100644 --- a/pkg/services/container/announcement/load/storage/storage_test.go +++ b/pkg/services/container/announcement/load/storage/storage_test.go @@ -12,8 +12,8 @@ import ( func TestStorage(t *testing.T) { const epoch uint64 = 13 - a := container.NewAnnouncement() - a.SetContainerID(cidtest.ID()) + var a container.SizeEstimation + a.SetContainer(cidtest.ID()) a.SetEpoch(epoch) const opinionsNum = 100 @@ -24,25 +24,23 @@ func TestStorage(t *testing.T) { for i := range opinions { opinions[i] = rand.Uint64() - a.SetUsedSpace(opinions[i]) + a.SetValue(opinions[i]) - require.NoError(t, s.Put(*a)) + require.NoError(t, s.Put(a)) } iterCounter := 0 err := s.Iterate( - func(ai container.UsedSpaceAnnouncement) bool { + func(ai container.SizeEstimation) bool { return ai.Epoch() == epoch }, - func(ai container.UsedSpaceAnnouncement) error { + func(ai container.SizeEstimation) error { iterCounter++ require.Equal(t, epoch, ai.Epoch()) - cnr1, _ := a.ContainerID() - cnr2, _ := ai.ContainerID() - require.Equal(t, cnr1, cnr2) - require.Equal(t, finalEstimation(opinions), ai.UsedSpace()) + require.Equal(t, a.Container(), ai.Container()) + require.Equal(t, finalEstimation(opinions), ai.Value()) return nil }, diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 7a7e5bed9..df8b4be97 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -11,7 +11,6 @@ import ( containercore "github.com/nspcc-dev/neofs-node/pkg/core/container" containerSvc "github.com/nspcc-dev/neofs-node/pkg/services/container" "github.com/nspcc-dev/neofs-node/pkg/services/object/acl/eacl" - containerSDK "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" eaclSDK "github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/session" @@ -58,8 +57,16 @@ func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *con return nil, errors.New("missing signature") } - cnr := containercore.Container{ - Value: containerSDK.NewContainerFromV2(body.GetContainer()), + cnrV2 := body.GetContainer() + if cnrV2 == nil { + return nil, errors.New("missing container field") + } + + var cnr containercore.Container + + err := cnr.Value.ReadFromV2(*cnrV2) + if err != nil { + return nil, fmt.Errorf("invalid container: %w", err) } cnr.Signature.ReadFromV2(*sigV2) @@ -158,8 +165,11 @@ func (s *morphExecutor) Get(ctx context.Context, body *container.GetRequestBody) cnr.Session.WriteToV2(tokV2) } + var cnrV2 container.Container + cnr.Value.WriteToV2(&cnrV2) + res := new(container.GetResponseBody) - res.SetContainer(cnr.Value.ToV2()) + res.SetContainer(&cnrV2) res.SetSignature(sigV2) res.SetSessionToken(tokV2) diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index 1c4b464dc..bb6e0a973 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -12,6 +12,7 @@ import ( containerSvcMorph "github.com/nspcc-dev/neofs-node/pkg/services/container/morph" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" + containertest "github.com/nspcc-dev/neofs-sdk-go/container/test" sessiontest "github.com/nspcc-dev/neofs-sdk-go/session/test" "github.com/stretchr/testify/require" ) @@ -54,6 +55,13 @@ func TestInvalidToken(t *testing.T) { var reqBody container.PutRequestBody reqBody.SetSignature(new(refs.Signature)) + cnr := containertest.Container() + + var cnrV2 container.Container + cnr.WriteToV2(&cnrV2) + + reqBody.SetContainer(&cnrV2) + _, err = e.Put(context.TODO(), tokV2, &reqBody) return }, diff --git a/pkg/services/object/acl/v2/classifier.go b/pkg/services/object/acl/v2/classifier.go index 99fe2a593..4888de221 100644 --- a/pkg/services/object/acl/v2/classifier.go +++ b/pkg/services/object/acl/v2/classifier.go @@ -3,7 +3,6 @@ package v2 import ( "bytes" "crypto/sha256" - "errors" core "github.com/nspcc-dev/neofs-node/pkg/core/netmap" "github.com/nspcc-dev/neofs-sdk-go/container" @@ -27,12 +26,7 @@ type classifyResult struct { func (c senderClassifier) classify( req MetaWithToken, idCnr cid.ID, - cnr *container.Container) (res *classifyResult, err error) { - ownerCnr := cnr.OwnerID() - if ownerCnr == nil { - return nil, errors.New("missing container owner") - } - + cnr container.Container) (res *classifyResult, err error) { ownerID, ownerKey, err := req.RequestOwner() if err != nil { return nil, err @@ -43,7 +37,7 @@ func (c senderClassifier) classify( // TODO: #767 get owner from neofs.id if present // if request owner is the same as container owner, return RoleUser - if ownerID.Equals(*ownerCnr) { + if ownerID.Equals(cnr.Owner()) { return &classifyResult{ role: acl.RoleOwner, key: ownerKeyInBytes, @@ -104,7 +98,7 @@ func (c senderClassifier) isInnerRingKey(owner []byte) (bool, error) { func (c senderClassifier) isContainerKey( owner, idCnr []byte, - cnr *container.Container) (bool, error) { + cnr container.Container) (bool, error) { nm, err := core.GetLatestNetworkMap(c.netmap) // first check current netmap if err != nil { return false, err @@ -130,13 +124,8 @@ func (c senderClassifier) isContainerKey( func lookupKeyInContainer( nm *netmap.NetMap, owner, idCnr []byte, - cnr *container.Container) (bool, error) { - policy := cnr.PlacementPolicy() - if policy == nil { - return false, errors.New("missing placement policy in container") - } - - cnrVectors, err := nm.ContainerNodes(*policy, idCnr) + cnr container.Container) (bool, error) { + cnrVectors, err := nm.ContainerNodes(cnr.PlacementPolicy(), idCnr) if err != nil { return false, err } diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index ac3a8931c..33d04f120 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -506,8 +506,6 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in cnr, err := b.containers.Get(idCnr) // fetch actual container if err != nil { return info, err - } else if cnr.Value.OwnerID() == nil { - return info, errors.New("missing owner in container descriptor") } if req.token != nil { @@ -534,7 +532,7 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in info.basicACL = cnr.Value.BasicACL() info.requestRole = res.role info.operation = op - info.cnrOwner = *cnr.Value.OwnerID() + info.cnrOwner = cnr.Value.Owner() info.idCnr = idCnr // it is assumed that at the moment the key will be valid, diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index aad3686ec..2fb0a1f30 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -35,7 +35,7 @@ type testStorage struct { } type testTraverserGenerator struct { - c *container.Container + c container.Container b map[uint64]placement.Builder } @@ -69,7 +69,7 @@ func newTestStorage() *testStorage { } func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e uint64) (*placement.Traverser, error) { - opts := make([]placement.Option, 3, 4) + opts := make([]placement.Option, 0, 4) opts = append(opts, placement.ForContainer(g.c), placement.UseBuilder(g.b[e]), @@ -466,9 +466,11 @@ func generateChain(ln int, cnr cid.ID) ([]*objectSDK.Object, []oid.ID, []byte) { func TestGetRemoteSmall(t *testing.T) { ctx := context.Background() - pp := netmaptest.PlacementPolicy() - cnr := container.New(container.WithPolicy(&pp)) - idCnr := container.CalculateID(cnr) + var cnr container.Container + cnr.SetPlacementPolicy(netmaptest.PlacementPolicy()) + + var idCnr cid.ID + container.CalculateID(&idCnr, cnr) newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { svc := &Service{cfg: new(cfg)} @@ -1119,9 +1121,11 @@ func TestGetRemoteSmall(t *testing.T) { func TestGetFromPastEpoch(t *testing.T) { ctx := context.Background() - pp := netmaptest.PlacementPolicy() - cnr := container.New(container.WithPolicy(&pp)) - idCnr := container.CalculateID(cnr) + var cnr container.Container + cnr.SetPlacementPolicy(netmaptest.PlacementPolicy()) + + var idCnr cid.ID + container.CalculateID(&idCnr, cnr) addr := oidtest.Address() addr.SetContainer(idCnr) diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 7ce3a4c52..a498a6391 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -33,7 +33,7 @@ type testStorage struct { } type testTraverserGenerator struct { - c *container.Container + c container.Container b map[uint64]placement.Builder } @@ -241,8 +241,11 @@ func TestGetRemoteSmall(t *testing.T) { var pp netmap.PlacementPolicy pp.AddReplicas(rs...) - cnr := container.New(container.WithPolicy(&pp)) - id := container.CalculateID(cnr) + var cnr container.Container + cnr.SetPlacementPolicy(pp) + + var id cid.ID + container.CalculateID(&id, cnr) newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { svc := &Service{cfg: new(cfg)} @@ -327,8 +330,11 @@ func TestGetFromPastEpoch(t *testing.T) { var pp netmap.PlacementPolicy pp.AddReplicas(rs...) - cnr := container.New(container.WithPolicy(&pp)) - idCnr := container.CalculateID(cnr) + var cnr container.Container + cnr.SetPlacementPolicy(pp) + + var idCnr cid.ID + container.CalculateID(&idCnr, cnr) var addr oid.Address addr.SetContainer(idCnr) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 7ccf40470..66a56d1e1 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -45,7 +45,8 @@ type cfg struct { obj *oid.ID - policy *netmap.PlacementPolicy + policySet bool + policy netmap.PlacementPolicy builder Builder } @@ -74,11 +75,11 @@ func NewTraverser(opts ...Option) (*Traverser, error) { if cfg.builder == nil { return nil, fmt.Errorf("%s: %w", invalidOptsMsg, errNilBuilder) - } else if cfg.policy == nil { + } else if !cfg.policySet { return nil, fmt.Errorf("%s: %w", invalidOptsMsg, errNilPolicy) } - ns, err := cfg.builder.BuildPlacement(cfg.cnr, cfg.obj, *cfg.policy) + ns, err := cfg.builder.BuildPlacement(cfg.cnr, cfg.obj, cfg.policy) if err != nil { return nil, fmt.Errorf("could not build placement: %w", err) } @@ -219,10 +220,11 @@ func UseBuilder(b Builder) Option { } // ForContainer is a traversal container setting option. -func ForContainer(cnr *container.Container) Option { +func ForContainer(cnr container.Container) Option { return func(c *cfg) { c.policy = cnr.PlacementPolicy() - c.cnr = container.CalculateID(cnr) + c.policySet = true + container.CalculateID(&c.cnr, cnr) } } diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 0dc46c05e..408173606 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -40,7 +40,7 @@ func copyVectors(v [][]netmap.NodeInfo) [][]netmap.NodeInfo { return vc } -func testPlacement(t *testing.T, ss, rs []int) ([][]netmap.NodeInfo, *container.Container) { +func testPlacement(t *testing.T, ss, rs []int) ([][]netmap.NodeInfo, container.Container) { nodes := make([][]netmap.NodeInfo, 0, len(rs)) replicas := make([]netmap.ReplicaDescriptor, 0, len(rs)) num := uint32(0) @@ -61,10 +61,13 @@ func testPlacement(t *testing.T, ss, rs []int) ([][]netmap.NodeInfo, *container. replicas = append(replicas, rd) } - policy := new(netmap.PlacementPolicy) + var policy netmap.PlacementPolicy policy.AddReplicas(replicas...) - return nodes, container.New(container.WithPolicy(policy)) + var cnr container.Container + cnr.SetPlacementPolicy(policy) + + return nodes, cnr } func assertSameAddress(t *testing.T, ni netmap.NodeInfo, addr network.AddressGroup) { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index d5de43b93..998d4d8a7 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -45,17 +45,9 @@ func (p *Policer) processObject(ctx context.Context, addr oid.Address) { } policy := cnr.Value.PlacementPolicy() - if policy == nil { - p.log.Error("missing placement policy in container", - zap.Stringer("cid", idCnr), - ) - - return - } - obj := addr.Object() - nn, err := p.placementBuilder.BuildPlacement(idCnr, &obj, *policy) + nn, err := p.placementBuilder.BuildPlacement(idCnr, &obj, policy) if err != nil { p.log.Error("could not build placement vector for object", zap.String("error", err.Error()),