From f0749fd23ef62c777892898c6b9b7f04f3f1b95c Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 15 Jun 2022 22:31:41 +0300 Subject: [PATCH] [#537] Upgrade NeoFS SDK Go with changed `netmap` package `PlacementPolicy` type now provides methods to work with QL-encoded policies. System network parameters can be read using dedicated method without iterating. Applications can work with `PlacementPolicy` variables directly so there is no need to use pointers. Signed-off-by: Leonard Lyubich --- api/handler/api.go | 2 +- api/handler/put.go | 4 ++- api/layer/container.go | 2 +- api/layer/layer.go | 2 +- authmate/authmate.go | 32 ++++++++++----------- cmd/s3-gw/app.go | 3 +- creds/accessbox/accessbox.go | 13 ++++----- go.mod | 2 +- go.sum | 4 +-- internal/neofs/neofs.go | 55 ++++++++---------------------------- 10 files changed, 43 insertions(+), 76 deletions(-) diff --git a/api/handler/api.go b/api/handler/api.go index 27bc5c29..46586463 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -24,7 +24,7 @@ type ( // Config contains data which handler needs to keep. Config struct { - DefaultPolicy *netmap.PlacementPolicy + DefaultPolicy netmap.PlacementPolicy DefaultMaxAge int NotificatorEnabled bool } diff --git a/api/handler/put.go b/api/handler/put.go index cb585a5e..51bcab49 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -623,16 +623,18 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) { return } + useDefaultPolicy := true if createParams.LocationConstraint != "" { for _, placementPolicy := range policies { if placementPolicy.LocationConstraint == createParams.LocationConstraint { p.Policy = placementPolicy.Policy p.LocationConstraint = createParams.LocationConstraint + useDefaultPolicy = false break } } } - if p.Policy == nil { + if useDefaultPolicy { p.Policy = h.cfg.DefaultPolicy } diff --git a/api/layer/container.go b/api/layer/container.go index 33d4d74e..9c3d6bf5 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -145,7 +145,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{ Creator: bktInfo.Owner, - Policy: *p.Policy, + Policy: p.Policy, Name: p.Name, SessionToken: p.SessionToken, AdditionalAttributes: attributes, diff --git a/api/layer/layer.go b/api/layer/layer.go index bed9bda8..3d8ee417 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -136,7 +136,7 @@ type ( // CreateBucketParams stores bucket create request parameters. CreateBucketParams struct { Name string - Policy *netmap.PlacementPolicy + Policy netmap.PlacementPolicy EACL *eacl.Table SessionToken *session.Container LocationConstraint string diff --git a/authmate/authmate.go b/authmate/authmate.go index b7b4bf29..9ed639f8 100644 --- a/authmate/authmate.go +++ b/authmate/authmate.go @@ -5,6 +5,7 @@ import ( "crypto/ecdsa" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -21,7 +22,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/netmap" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/user" "go.uber.org/zap" @@ -147,16 +147,17 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn return opts.ID, a.neoFS.ContainerExists(ctx, *opts.ID) } - pp, err := policy.Parse(opts.PlacementPolicy) + var prm PrmContainerCreate + + err := prm.Policy.DecodeString(opts.PlacementPolicy) if err != nil { return nil, fmt.Errorf("failed to build placement policy: %w", err) } - cnrID, err := a.neoFS.CreateContainer(ctx, PrmContainerCreate{ - Owner: idOwner, - Policy: *pp, - FriendlyName: opts.FriendlyName, - }) + prm.Owner = idOwner + prm.FriendlyName = opts.FriendlyName + + cnrID, err := a.neoFS.CreateContainer(ctx, prm) if err != nil { return nil, err } @@ -165,17 +166,18 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn } func checkPolicy(policyString string) (*netmap.PlacementPolicy, error) { - result, err := policy.Parse(policyString) + var result netmap.PlacementPolicy + + err := result.DecodeString(policyString) if err == nil { - return result, nil + return &result, nil } - result = netmap.NewPlacementPolicy() if err = result.UnmarshalJSON([]byte(policyString)); err == nil { - return result, nil + return &result, nil } - return nil, fmt.Errorf("can't parse placement policy") + return nil, errors.New("can't parse placement policy") } func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPolicy, error) { @@ -189,14 +191,10 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo if err != nil { return nil, err } - marshaled, err := parsedPolicy.Marshal() - if err != nil { - return nil, fmt.Errorf("can't marshal placement policy: %w", err) - } result = append(result, &accessbox.AccessBox_ContainerPolicy{ LocationConstraint: locationConstraint, - Policy: marshaled, + Policy: parsedPolicy.Marshal(), }) } diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 564eafeb..49811a83 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -20,7 +20,6 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/internal/neofs" "github.com/nspcc-dev/neofs-s3-gw/internal/version" "github.com/nspcc-dev/neofs-s3-gw/internal/wallet" - "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "go.uber.org/zap" @@ -368,7 +367,7 @@ func getHandlerOptions(v *viper.Viper, l *zap.Logger) *handler.Config { policyStr = v.GetString(cfgDefaultPolicy) } - if cfg.DefaultPolicy, err = policy.Parse(policyStr); err != nil { + if err = cfg.DefaultPolicy.DecodeString(policyStr); err != nil { l.Fatal("couldn't parse container default policy", zap.Error(err)) } diff --git a/creds/accessbox/accessbox.go b/creds/accessbox/accessbox.go index cd5fce7e..7ff85bb9 100644 --- a/creds/accessbox/accessbox.go +++ b/creds/accessbox/accessbox.go @@ -28,7 +28,7 @@ type Box struct { // ContainerPolicy represents friendly AccessBox_ContainerPolicy. type ContainerPolicy struct { LocationConstraint string - Policy *netmap.PlacementPolicy + Policy netmap.PlacementPolicy } // GateData represents gate tokens in AccessBox. @@ -141,15 +141,14 @@ func (x *AccessBox) GetTokens(owner *keys.PrivateKey) (*GateData, error) { func (x *AccessBox) GetPlacementPolicy() ([]*ContainerPolicy, error) { var result []*ContainerPolicy for _, policy := range x.ContainerPolicy { - placementPolicy := netmap.NewPlacementPolicy() - if err := placementPolicy.Unmarshal(policy.Policy); err != nil { + var cnrPolicy ContainerPolicy + if err := cnrPolicy.Policy.Unmarshal(policy.Policy); err != nil { return nil, err } - result = append(result, &ContainerPolicy{ - LocationConstraint: policy.LocationConstraint, - Policy: placementPolicy, - }) + cnrPolicy.LocationConstraint = policy.LocationConstraint + + result = append(result, &cnrPolicy) } return result, nil diff --git a/go.mod b/go.mod index 543bf1a8..b9341335 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d github.com/nspcc-dev/neo-go v0.98.2 github.com/nspcc-dev/neofs-api-go/v2 v2.12.2 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721 github.com/prometheus/client_golang v1.11.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 diff --git a/go.sum b/go.sum index 73cdecb4..c52c2c84 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnB github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6 h1:aQldkeCRphHVSmsPD+05+CwLPImuLwySJkigL3AiTu8= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6/go.mod h1:k58jgszGX3pws2yiOXu9m0i32BzRgi1T6Bpd/L1KrJU= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721 h1:5Al3dddr0SG3ONhfglTyc2GSnQS0vMmygCD00vLo/jU= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721/go.mod h1:k58jgszGX3pws2yiOXu9m0i32BzRgi1T6Bpd/L1KrJU= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/internal/neofs/neofs.go b/internal/neofs/neofs.go index d12828c8..341c5262 100644 --- a/internal/neofs/neofs.go +++ b/internal/neofs/neofs.go @@ -3,7 +3,6 @@ package neofs import ( "bytes" "context" - "encoding/binary" "errors" "fmt" "io" @@ -21,7 +20,6 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/eacl" - "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" @@ -68,22 +66,7 @@ func (x *NeoFS) TimeToEpoch(ctx context.Context, futureTime time.Time) (uint64, return 0, 0, fmt.Errorf("get network info via client: %w", err) } - var durEpoch uint64 - - networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { - if string(parameter.Key()) == "EpochDuration" { - data := make([]byte, 8) - - copy(data, parameter.Value()) - - durEpoch = binary.LittleEndian.Uint64(data) - - return true - } - - return false - }) - + durEpoch := networkInfo.EpochDuration() if durEpoch == 0 { return 0, 0, errors.New("epoch duration is missing or zero") } @@ -509,22 +492,12 @@ func (x *ResolverNeoFS) SystemDNS(ctx context.Context) (string, error) { return "", fmt.Errorf("read network info via client: %w", err) } - var domain string - - networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { - if string(parameter.Key()) == "SystemDNS" { - domain = string(parameter.Value()) - return true - } - - return false - }) - - if domain == "" { + domain := networkInfo.RawNetworkParameter("SystemDNS") + if domain == nil { return "", errors.New("system DNS parameter not found or empty") } - return domain, nil + return string(domain), nil } // AuthmateNeoFS is a mediator which implements authmate.NeoFS through pool.Pool. @@ -590,22 +563,18 @@ func (x *AuthmateNeoFS) CreateObject(ctx context.Context, prm tokens.PrmObjectCr }) } -func isHomomorphicHashDisabled(ctx context.Context, p *pool.Pool) (res bool, err error) { +func isHomomorphicHashDisabled(ctx context.Context, p *pool.Pool) (bool, error) { ni, err := p.NetworkInfo(ctx) if err != nil { return false, err } - var expectedParamKey = []byte("HomomorphicHashingDisabled") + // FIXME(@cthulhu-rider): parameter format hasn't been fixed in the protocol yet, + // use decoding provided by SDK after neofs-api#214 support. + rawPrm := ni.RawNetworkParameter("HomomorphicHashingDisabled") + if rawPrm != nil { + return stackitem.NewByteArray(rawPrm).TryBool() + } - ni.NetworkConfig().IterateParameters(func(p *netmap.NetworkParameter) bool { - if bytes.Equal(p.Key(), expectedParamKey) { - arr := stackitem.NewByteArray(p.Value()) - res, err = arr.TryBool() - return true - } - return false - }) - - return res, nil + return false, nil }