TrueCloudLab/frostfs-s3-gw
18
3
Fork 20
Code Issues 29 Pull requests 8 Projects Releases 23 Packages Activity Actions

[#537] Upgrade NeoFS SDK Go with changed netmap package

Browse source 
`PlacementPolicy` type now provides methods to work with QL-encoded
policies. System network parameters can be read using dedicated method
without iterating. Applications can work with `PlacementPolicy`
variables directly so there is no need to use pointers.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-06-15 22:31:41 +03:00 committed by Alex Vanin
parent cfe7591cf7
commit f0749fd23e
10 changed files with 43 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/handler/api.go
View file

@ -24,7 +24,7 @@ type (
// Config contains data which handler needs to keep.
Config struct {
DefaultPolicy *netmap.PlacementPolicy
DefaultPolicy netmap.PlacementPolicy
DefaultMaxAge int
NotificatorEnabled bool
}

4
api/handler/put.go
View file

@ -623,16 +623,18 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
useDefaultPolicy := true
if createParams.LocationConstraint != "" {
for _, placementPolicy := range policies {
if placementPolicy.LocationConstraint == createParams.LocationConstraint {
p.Policy = placementPolicy.Policy
p.LocationConstraint = createParams.LocationConstraint
useDefaultPolicy = false
break
}
}
}
if p.Policy == nil {
if useDefaultPolicy {
p.Policy = h.cfg.DefaultPolicy
}

2
api/layer/container.go
View file

@ -145,7 +145,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{
Creator: bktInfo.Owner,
Policy: *p.Policy,
Policy: p.Policy,
Name: p.Name,
SessionToken: p.SessionToken,
AdditionalAttributes: attributes,

2
api/layer/layer.go
View file

@ -136,7 +136,7 @@ type (
// CreateBucketParams stores bucket create request parameters.
CreateBucketParams struct {
Name string
Policy *netmap.PlacementPolicy
Policy netmap.PlacementPolicy
EACL *eacl.Table
SessionToken *session.Container
LocationConstraint string

32
authmate/authmate.go
View file

@ -5,6 +5,7 @@ import (
"crypto/ecdsa"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
@ -21,7 +22,6 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
@ -147,16 +147,17 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
return opts.ID, a.neoFS.ContainerExists(ctx, *opts.ID)
}
pp, err := policy.Parse(opts.PlacementPolicy)
var prm PrmContainerCreate
err := prm.Policy.DecodeString(opts.PlacementPolicy)
if err != nil {
return nil, fmt.Errorf("failed to build placement policy: %w", err)
}
cnrID, err := a.neoFS.CreateContainer(ctx, PrmContainerCreate{
Owner: idOwner,
Policy: *pp,
FriendlyName: opts.FriendlyName,
})
prm.Owner = idOwner
prm.FriendlyName = opts.FriendlyName
cnrID, err := a.neoFS.CreateContainer(ctx, prm)
if err != nil {
return nil, err
}
@ -165,17 +166,18 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
}
func checkPolicy(policyString string) (*netmap.PlacementPolicy, error) {
result, err := policy.Parse(policyString)
var result netmap.PlacementPolicy
err := result.DecodeString(policyString)
if err == nil {
return result, nil
return &result, nil
}
result = netmap.NewPlacementPolicy()
if err = result.UnmarshalJSON([]byte(policyString)); err == nil {
return result, nil
return &result, nil
}
return nil, fmt.Errorf("can't parse placement policy")
return nil, errors.New("can't parse placement policy")
}
func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPolicy, error) {
@ -189,14 +191,10 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo
if err != nil {
return nil, err
}
marshaled, err := parsedPolicy.Marshal()
if err != nil {
return nil, fmt.Errorf("can't marshal placement policy: %w", err)
}
result = append(result, &accessbox.AccessBox_ContainerPolicy{
LocationConstraint: locationConstraint,
Policy: marshaled,
Policy: parsedPolicy.Marshal(),
})
}

3
cmd/s3-gw/app.go
View file

@ -20,7 +20,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
"github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/spf13/viper"
"go.uber.org/zap"
@ -368,7 +367,7 @@ func getHandlerOptions(v *viper.Viper, l *zap.Logger) *handler.Config {
policyStr = v.GetString(cfgDefaultPolicy)
}
if cfg.DefaultPolicy, err = policy.Parse(policyStr); err != nil {
if err = cfg.DefaultPolicy.DecodeString(policyStr); err != nil {
l.Fatal("couldn't parse container default policy",
zap.Error(err))
}

13
creds/accessbox/accessbox.go
View file

@ -28,7 +28,7 @@ type Box struct {
// ContainerPolicy represents friendly AccessBox_ContainerPolicy.
type ContainerPolicy struct {
LocationConstraint string
Policy *netmap.PlacementPolicy
Policy netmap.PlacementPolicy
}
// GateData represents gate tokens in AccessBox.
@ -141,15 +141,14 @@ func (x *AccessBox) GetTokens(owner *keys.PrivateKey) (*GateData, error) {
func (x *AccessBox) GetPlacementPolicy() ([]*ContainerPolicy, error) {
var result []*ContainerPolicy
for _, policy := range x.ContainerPolicy {
placementPolicy := netmap.NewPlacementPolicy()
if err := placementPolicy.Unmarshal(policy.Policy); err != nil {
var cnrPolicy ContainerPolicy
if err := cnrPolicy.Policy.Unmarshal(policy.Policy); err != nil {
return nil, err
}
result = append(result, &ContainerPolicy{
LocationConstraint: policy.LocationConstraint,
Policy: placementPolicy,
})
cnrPolicy.LocationConstraint = policy.LocationConstraint
result = append(result, &cnrPolicy)
}
return result, nil

2
go.mod
View file

@ -10,7 +10,7 @@ require (
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d
github.com/nspcc-dev/neo-go v0.98.2
github.com/nspcc-dev/neofs-api-go/v2 v2.12.2
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721
github.com/prometheus/client_golang v1.11.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1

4
go.sum
View file

@ -306,8 +306,8 @@ github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnB
github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4=
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6 h1:aQldkeCRphHVSmsPD+05+CwLPImuLwySJkigL3AiTu8=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220607185339-031eac2f48f6/go.mod h1:k58jgszGX3pws2yiOXu9m0i32BzRgi1T6Bpd/L1KrJU=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721 h1:5Al3dddr0SG3ONhfglTyc2GSnQS0vMmygCD00vLo/jU=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.4.0.20220616082321-e986f4780721/go.mod h1:k58jgszGX3pws2yiOXu9m0i32BzRgi1T6Bpd/L1KrJU=
github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=

55
internal/neofs/neofs.go
View file

@ -3,7 +3,6 @@ package neofs
import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
@ -21,7 +20,6 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/pool"
@ -68,22 +66,7 @@ func (x *NeoFS) TimeToEpoch(ctx context.Context, futureTime time.Time) (uint64,
return 0, 0, fmt.Errorf("get network info via client: %w", err)
}
var durEpoch uint64
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "EpochDuration" {
data := make([]byte, 8)
copy(data, parameter.Value())
durEpoch = binary.LittleEndian.Uint64(data)
return true
}
return false
})
durEpoch := networkInfo.EpochDuration()
if durEpoch == 0 {
return 0, 0, errors.New("epoch duration is missing or zero")
}
@ -509,22 +492,12 @@ func (x *ResolverNeoFS) SystemDNS(ctx context.Context) (string, error) {
return "", fmt.Errorf("read network info via client: %w", err)
}
var domain string
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "SystemDNS" {
domain = string(parameter.Value())
return true
}
return false
})
if domain == "" {
domain := networkInfo.RawNetworkParameter("SystemDNS")
if domain == nil {
return "", errors.New("system DNS parameter not found or empty")
}
return domain, nil
return string(domain), nil
}
// AuthmateNeoFS is a mediator which implements authmate.NeoFS through pool.Pool.
@ -590,22 +563,18 @@ func (x *AuthmateNeoFS) CreateObject(ctx context.Context, prm tokens.PrmObjectCr
})
}
func isHomomorphicHashDisabled(ctx context.Context, p *pool.Pool) (res bool, err error) {
func isHomomorphicHashDisabled(ctx context.Context, p *pool.Pool) (bool, error) {
ni, err := p.NetworkInfo(ctx)
if err != nil {
return false, err
}
var expectedParamKey = []byte("HomomorphicHashingDisabled")
// FIXME(@cthulhu-rider): parameter format hasn't been fixed in the protocol yet,
// use decoding provided by SDK after neofs-api#214 support.
rawPrm := ni.RawNetworkParameter("HomomorphicHashingDisabled")
if rawPrm != nil {
return stackitem.NewByteArray(rawPrm).TryBool()
}
ni.NetworkConfig().IterateParameters(func(p *netmap.NetworkParameter) bool {
if bytes.Equal(p.Key(), expectedParamKey) {
arr := stackitem.NewByteArray(p.Value())
res, err = arr.TryBool()
return true
}
return false
})
return res, nil
return false, nil
}