nzinkevich/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#346] *: Refactor communication with NeoFS at the protocol level

Browse source 
Make `tokens`, `authmate` and `layer` packages to depend from locally
defined `NeoFS` interface of the virtual connection to NeoFS network.
Create internal `neofs` package and implement these interfaces through
`pool.Pool` there. Implement mediators between `NeoFS` interfaces and
`neofs.NeoFS` implementation.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-03-01 22:02:24 +03:00 committed by LeL
parent 34a221c5c9
commit cd64f41ce8
14 changed files with 1348 additions and 606 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api/auth/center.go
View file

@ -22,8 +22,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/object/address"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"go.uber.org/zap"
) )
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. // authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
@ -44,12 +42,6 @@ type (
cli tokens.Credentials cli tokens.Credentials
} }
// Params stores node connection parameters.
Params struct {
Pool pool.Pool
Logger *zap.Logger
}
prs int prs int
authHeader struct { authHeader struct {
@ -82,9 +74,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
var _ io.ReadSeeker = prs(0) var _ io.ReadSeeker = prs(0)
// New creates an instance of AuthCenter. // New creates an instance of AuthCenter.
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Center { func New(neoFS tokens.NeoFS, key *keys.PrivateKey, config *cache.Config) Center {
return &center{ return &center{
cli: tokens.New(conns, key, config), cli: tokens.New(neoFS, key, config),
reg: &regexpSubmatcher{re: authorizationFieldRegexp}, reg: &regexpSubmatcher{re: authorizationFieldRegexp},
postReg: &regexpSubmatcher{re: postPolicyCredentialRegexp}, postReg: &regexpSubmatcher{re: postPolicyCredentialRegexp},
} }

82
api/layer/container.go
View file

@ -15,7 +15,6 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/container" "github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -30,21 +29,21 @@ type (
const locationConstraintAttr = ".s3-location-constraint" const locationConstraintAttr = ".s3-location-constraint"
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInfo, error) { func (n *layer) containerInfo(ctx context.Context, idCnr *cid.ID) (*data.BucketInfo, error) {
var ( var (
err error err error
res *container.Container res *container.Container
rid = api.GetRequestID(ctx) rid = api.GetRequestID(ctx)
info = &data.BucketInfo{ info = &data.BucketInfo{
CID: cid, CID: idCnr,
Name: cid.String(), Name: idCnr.String(),
} }
) )
res, err = n.pool.GetContainer(ctx, cid, n.CallOptions(ctx)...) res, err = n.neoFS.Container(ctx, *idCnr)
if err != nil { if err != nil {
n.log.Error("could not fetch container", n.log.Error("could not fetch container",
zap.Stringer("cid", cid), zap.Stringer("cid", idCnr),
zap.String("request_id", rid), zap.String("request_id", rid),
zap.Error(err)) zap.Error(err))
@ -65,7 +64,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInf
unix, err := strconv.ParseInt(attr.Value(), 10, 64) unix, err := strconv.ParseInt(attr.Value(), 10, 64)
if err != nil { if err != nil {
n.log.Error("could not parse container creation time", n.log.Error("could not parse container creation time",
zap.Stringer("cid", cid), zap.Stringer("cid", idCnr),
zap.String("request_id", rid), zap.String("request_id", rid),
zap.String("created_at", val), zap.String("created_at", val),
zap.Error(err)) zap.Error(err))
@ -81,7 +80,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInf
if err := n.bucketCache.Put(info); err != nil { if err := n.bucketCache.Put(info); err != nil {
n.log.Warn("could not put bucket info into cache", n.log.Warn("could not put bucket info into cache",
zap.Stringer("cid", cid), zap.Stringer("cid", idCnr),
zap.String("bucket_name", info.Name), zap.String("bucket_name", info.Name),
zap.Error(err)) zap.Error(err))
} }
@ -93,20 +92,20 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
var ( var (
err error err error
own = n.Owner(ctx) own = n.Owner(ctx)
res []*cid.ID res []cid.ID
rid = api.GetRequestID(ctx) rid = api.GetRequestID(ctx)
) )
res, err = n.pool.ListContainers(ctx, own, n.CallOptions(ctx)...) res, err = n.neoFS.UserContainers(ctx, *own)
if err != nil { if err != nil {
n.log.Error("could not fetch container", n.log.Error("could not list user containers",
zap.String("request_id", rid), zap.String("request_id", rid),
zap.Error(err)) zap.Error(err))
return nil, err return nil, err
} }
list := make([]*data.BucketInfo, 0, len(res)) list := make([]*data.BucketInfo, 0, len(res))
for _, cid := range res { for i := range res {
info, err := n.containerInfo(ctx, cid) info, err := n.containerInfo(ctx, &res[i])
if err != nil { if err != nil {
n.log.Error("could not fetch container info", n.log.Error("could not fetch container info",
zap.String("request_id", rid), zap.String("request_id", rid),
@ -130,28 +129,23 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*ci
LocationConstraint: p.LocationConstraint, LocationConstraint: p.LocationConstraint,
} }
options := []container.Option{ var locConstAttr *container.Attribute
container.WithPolicy(p.Policy),
container.WithCustomBasicACL(acl.BasicACL(p.ACL)),
container.WithAttribute(container.AttributeName, p.Name),
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(bktInfo.Created.Unix(), 10)),
}
if p.LocationConstraint != "" { if p.LocationConstraint != "" {
options = append(options, container.WithAttribute(locationConstraintAttr, p.LocationConstraint)) locConstAttr = container.NewAttribute()
locConstAttr.SetKey(locationConstraintAttr)
locConstAttr.SetValue(p.LocationConstraint)
} }
cnr := container.New(options...) if bktInfo.CID, err = n.neoFS.CreateContainer(ctx, PrmContainerCreate{
container.SetNativeName(cnr, p.Name) Creator: *bktInfo.Owner,
Policy: *p.Policy,
cnr.SetSessionToken(p.SessionToken) Name: p.Name,
cnr.SetOwnerID(bktInfo.Owner) SessionToken: p.SessionToken,
Time: bktInfo.Created,
if bktInfo.CID, err = n.pool.PutContainer(ctx, cnr); err != nil { BasicACL: acl.BasicACL(p.ACL),
return nil, err LocationConstraintAttribute: locConstAttr,
} }); err != nil {
if err = n.pool.WaitForContainerPresence(ctx, bktInfo.CID, pool.DefaultPollingParams()); err != nil {
return nil, err return nil, err
} }
@ -172,21 +166,20 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*ci
func (n *layer) setContainerEACLTable(ctx context.Context, cid *cid.ID, table *eacl.Table) error { func (n *layer) setContainerEACLTable(ctx context.Context, cid *cid.ID, table *eacl.Table) error {
table.SetCID(cid) table.SetCID(cid)
var sessionToken *session.Token
boxData, err := GetBoxData(ctx) boxData, err := GetBoxData(ctx)
if err == nil { if err == nil {
sessionToken = boxData.Gate.SessionTokenForSetEACL() table.SetSessionToken(boxData.Gate.SessionTokenForSetEACL())
} }
if err := n.pool.SetEACL(ctx, table, pool.WithSession(sessionToken)); err != nil { if err := n.neoFS.SetContainerEACL(ctx, *table); err != nil {
return err return err
} }
return n.waitEACLPresence(ctx, cid, table, defaultWaitParams()) return n.waitEACLPresence(ctx, *cid, table, defaultWaitParams())
} }
func (n *layer) GetContainerEACL(ctx context.Context, cid *cid.ID) (*eacl.Table, error) { func (n *layer) GetContainerEACL(ctx context.Context, cid *cid.ID) (*eacl.Table, error) {
return n.pool.GetEACL(ctx, cid) return n.neoFS.ContainerEACL(ctx, *cid)
} }
type waitParams struct { type waitParams struct {
@ -201,7 +194,7 @@ func defaultWaitParams() *waitParams {
} }
} }
func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.Table, params *waitParams) error { func (n *layer) waitEACLPresence(ctx context.Context, cid cid.ID, table *eacl.Table, params *waitParams) error {
exp, err := table.Marshal() exp, err := table.Marshal()
if err != nil { if err != nil {
return fmt.Errorf("couldn't marshal eacl: %w", err) return fmt.Errorf("couldn't marshal eacl: %w", err)
@ -213,6 +206,8 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
defer ticker.Stop() defer ticker.Stop()
wdone := wctx.Done() wdone := wctx.Done()
done := ctx.Done() done := ctx.Done()
var eaclTable *eacl.Table
var got []byte
for { for {
select { select {
case <-done: case <-done:
@ -220,10 +215,13 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
case <-wdone: case <-wdone:
return wctx.Err() return wctx.Err()
case <-ticker.C: case <-ticker.C:
eaclTable, err := n.pool.GetEACL(ctx, cid) eaclTable, err = n.neoFS.ContainerEACL(ctx, cid)
if err == nil { if err == nil {
got, err := eaclTable.Marshal() got, err = eaclTable.Marshal()
if err == nil && bytes.Equal(exp, got) { if err != nil {
// not expected, but if occurred - doesn't make sense to continue
return fmt.Errorf("marshal received eACL: %w", err)
} else if bytes.Equal(exp, got) {
return nil return nil
} }
} }
@ -232,11 +230,11 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
} }
} }
func (n *layer) deleteContainer(ctx context.Context, cid *cid.ID) error { func (n *layer) deleteContainer(ctx context.Context, idCnr *cid.ID) error {
var sessionToken *session.Token var sessionToken *session.Token
boxData, err := GetBoxData(ctx) boxData, err := GetBoxData(ctx)
if err == nil { if err == nil {
sessionToken = boxData.Gate.SessionTokenForDelete() sessionToken = boxData.Gate.SessionTokenForDelete()
} }
return n.pool.DeleteContainer(ctx, cid, pool.WithSession(sessionToken)) return n.neoFS.DeleteContainer(ctx, *idCnr, sessionToken)
} }

249
api/layer/layer.go
View file

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"crypto/ecdsa" "crypto/ecdsa"
stderrors "errors"
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
@ -18,6 +19,8 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/notifications" "github.com/nspcc-dev/neofs-s3-gw/api/notifications"
"github.com/nspcc-dev/neofs-s3-gw/api/resolver" "github.com/nspcc-dev/neofs-s3-gw/api/resolver"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/acl"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/netmap"
@ -27,12 +30,216 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/token"
"go.uber.org/zap" "go.uber.org/zap"
) )
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
type PrmContainerCreate struct {
// NeoFS identifier of the container creator.
Creator owner.ID
// Container placement policy.
Policy netmap.PlacementPolicy
// Name for the container.
Name string
// Token of the container's creation session. Nil means session absence.
SessionToken *session.Token
// Time when container is created.
Time time.Time
// Basic ACL of the container.
BasicACL acl.BasicACL
// Attribute for LocationConstraint parameter (optional).
LocationConstraintAttribute *container.Attribute
}
// PrmAuth groups authentication parameters for the NeoFS operation.
type PrmAuth struct {
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
BearerToken *token.BearerToken
// Private key used for the operation if BearerToken is missing (in this case non-nil).
PrivateKey *ecdsa.PrivateKey
}
// PrmObjectSelect groups parameters of NeoFS.SelectObjects operation.
type PrmObjectSelect struct {
// Authentication parameters.
PrmAuth
// Container to select the objects from.
Container cid.ID
// Key-value object attribute which should exactly be
// presented in selected objects. Optional, empty key means any.
ExactAttribute [2]string
// File prefix of the selected objects. Optional, empty value means any.
FilePrefix string
}
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
type PrmObjectRead struct {
// Authentication parameters.
PrmAuth
// Container to read the object header from.
Container cid.ID
// ID of the object for which to read the header.
Object oid.ID
// Flag to read object header.
WithHeader bool
// Flag to read object payload. False overlaps payload range.
WithPayload bool
// Offset-length range of the object payload to be read.
PayloadRange [2]uint64
}
// ObjectPart represents partially read NeoFS object.
type ObjectPart struct {
// Object header with optional in-memory payload part.
Head *object.Object
// Object payload part encapsulated in io.Reader primitive.
// Returns ErrAccessDenied on read access violation.
Payload io.ReadCloser
}
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
type PrmObjectCreate struct {
// Authentication parameters.
PrmAuth
// Container to store the object.
Container cid.ID
// NeoFS identifier of the object creator.
Creator owner.ID
// Key-value object attributes.
Attributes [][2]string
// Full payload size (optional).
PayloadSize uint64
// Associated filename (optional).
Filename string
// Object payload encapsulated in io.Reader primitive.
Payload io.Reader
}
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
type PrmObjectDelete struct {
// Authentication parameters.
PrmAuth
// Container to delete the object from.
Container cid.ID
// Identifier of the removed object.
Object oid.ID
}
// ErrAccessDenied is returned from NeoFS in case of access violation.
var ErrAccessDenied = stderrors.New("access denied")
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// CreateContainer creates and saves parameterized container in NeoFS.
// Returns ID of the saved container.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the container to be created.
CreateContainer(context.Context, PrmContainerCreate) (*cid.ID, error)
// Container reads container from NeoFS by ID.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the container to be read.
Container(context.Context, cid.ID) (*container.Container, error)
// UserContainers reads list of the containers owned by specified user.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the containers to be listed.
UserContainers(context.Context, owner.ID) ([]cid.ID, error)
// SetContainerEACL saves eACL table of the container in NeoFS.
//
// Returns any error encountered which prevented the eACL to be saved.
SetContainerEACL(context.Context, eacl.Table) error
// ContainerEACL reads container eACL from NeoFS by container ID.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the eACL to be read.
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
// DeleteContainer marks the container to be removed from NeoFS by ID.
// Request is sent within session if the session token is specified.
// Successful return does not guarantee the actual removal.
//
// Returns any error encountered which prevented the removal request to be sent.
DeleteContainer(context.Context, cid.ID, *session.Token) error
// SelectObjects perform object selection from the NeoFS container according
// to specified parameters. Selects user objects only.
//
// Returns ErrAccessDenied on selection access violation.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the objects to be selected.
SelectObjects(context.Context, PrmObjectSelect) ([]oid.ID, error)
// ReadObject reads part of the object from the NeoFS container by identifier.
// Exact part is returned according to the parameters:
// * with header only: empty payload (both in-mem and reader parts are nil);
// * with payload only: header is nil (zero range means full payload);
// * with header and payload: full in-mem object, payload reader is nil.
//
// WithHeader or WithPayload is true. Range length is positive if offset is positive.
//
// Payload reader should be closed if it is no longer needed.
//
// Returns ErrAccessDenied on read access violation.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the object header to be read.
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
// CreateObject creates and saves parameterized object in the NeoFS container.
// Returns ID of the saved object.
//
// Creation time should be written into object (UTC).
//
// Returns ErrAccessDenied on write access violation.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the container to be created.
CreateObject(context.Context, PrmObjectCreate) (*oid.ID, error)
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
// Successful return does not guarantee the actual removal.
//
// Returns ErrAccessDenied on remove access violation.
//
// Returns any error encountered which prevented the removal request to be sent.
DeleteObject(context.Context, PrmObjectDelete) error
}
type ( type (
layer struct { layer struct {
pool pool.Pool neoFS NeoFS
log *zap.Logger log *zap.Logger
anonKey AnonymousKey anonKey AnonymousKey
resolver *resolver.BucketResolver resolver *resolver.BucketResolver
@ -66,14 +273,6 @@ type (
System *cache.Config System *cache.Config
} }
// Params stores basic API parameters.
Params struct {
Pool pool.Pool
Logger *zap.Logger
Timeout time.Duration
Key *ecdsa.PrivateKey
}
// GetObjectParams stores object get request parameters. // GetObjectParams stores object get request parameters.
GetObjectParams struct { GetObjectParams struct {
Range *RangeParams Range *RangeParams
@ -183,15 +382,8 @@ type (
TagSet map[string]string TagSet map[string]string
} }
// NeoFS provides basic NeoFS interface.
NeoFS interface {
Get(ctx context.Context, address *address.Address) (*object.Object, error)
}
// Client provides S3 API client interface. // Client provides S3 API client interface.
Client interface { Client interface {
NeoFS
EphemeralKey() *keys.PublicKey EphemeralKey() *keys.PublicKey
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error)
@ -262,9 +454,9 @@ func DefaultCachesConfigs() *CachesConfig {
// NewLayer creates instance of layer. It checks credentials // NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node. // and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, conns pool.Pool, config *Config) Client { func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
return &layer{ return &layer{
pool: conns, neoFS: neoFS,
log: log, log: log,
anonKey: config.AnonKey, anonKey: config.AnonKey,
resolver: config.Resolver, resolver: config.Resolver,
@ -293,17 +485,26 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
// Owner returns owner id from BearerToken (context) or from client owner. // Owner returns owner id from BearerToken (context) or from client owner.
func (n *layer) Owner(ctx context.Context) *owner.ID { func (n *layer) Owner(ctx context.Context) *owner.ID {
if data, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && data != nil && data.Gate != nil { if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
return data.Gate.BearerToken.Issuer() return bd.Gate.BearerToken.Issuer()
} }
return owner.NewIDFromPublicKey((*ecdsa.PublicKey)(n.EphemeralKey())) return owner.NewIDFromPublicKey((*ecdsa.PublicKey)(n.EphemeralKey()))
} }
func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth) {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
prm.BearerToken = bd.Gate.BearerToken
return
}
prm.PrivateKey = &n.anonKey.Key.PrivateKey
}
// CallOptions returns []pool.CallOption options: client.WithBearer or client.WithKey (if request is anonymous). // CallOptions returns []pool.CallOption options: client.WithBearer or client.WithKey (if request is anonymous).
func (n *layer) CallOptions(ctx context.Context) []pool.CallOption { func (n *layer) CallOptions(ctx context.Context) []pool.CallOption {
if data, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && data != nil && data.Gate != nil { if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
return []pool.CallOption{pool.WithBearer(data.Gate.BearerToken)} return []pool.CallOption{pool.WithBearer(bd.Gate.BearerToken)}
} }
return []pool.CallOption{pool.WithKey(&n.anonKey.Key.PrivateKey)} return []pool.CallOption{pool.WithKey(&n.anonKey.Key.PrivateKey)}
@ -341,14 +542,14 @@ func (n *layer) GetBucketACL(ctx context.Context, name string) (*BucketACL, erro
return nil, err return nil, err
} }
eacl, err := n.GetContainerEACL(ctx, inf.CID) eACL, err := n.GetContainerEACL(ctx, inf.CID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &BucketACL{ return &BucketACL{
Info: inf, Info: inf,
EACL: eacl, EACL: eACL,
}, nil }, nil
} }

4
api/layer/multipart_upload.go
View file

@ -308,8 +308,8 @@ func (n *layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUpload
} }
f := &findParams{ f := &findParams{
filters: []filter{{attr: UploadPartNumberAttributeName, val: "0"}}, attr: [2]string{UploadPartNumberAttributeName, "0"},
cid: p.Bkt.CID, cid: p.Bkt.CID,
} }
ids, err := n.objectSearch(ctx, f) ids, err := n.objectSearch(ctx, f)

215
api/layer/object.go
View file

@ -3,10 +3,8 @@ package layer
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"io" "io"
"sort" "sort"
"strconv"
"strings" "strings"
"time" "time"
@ -18,20 +16,14 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/owner"
"go.uber.org/zap" "go.uber.org/zap"
) )
type ( type (
findParams struct { findParams struct {
filters []filter attr [2]string
cid *cid.ID cid *cid.ID
prefix string prefix string
}
filter struct {
attr string
val string
} }
getParams struct { getParams struct {
@ -76,54 +68,25 @@ type (
func (n *layer) objectSearchByName(ctx context.Context, cid *cid.ID, filename string) ([]oid.ID, error) { func (n *layer) objectSearchByName(ctx context.Context, cid *cid.ID, filename string) ([]oid.ID, error) {
f := &findParams{ f := &findParams{
filters: []filter{{attr: object.AttributeFileName, val: filename}}, attr: [2]string{object.AttributeFileName, filename},
cid: cid, cid: cid,
prefix: "",
} }
return n.objectSearch(ctx, f) return n.objectSearch(ctx, f)
} }
// objectSearch returns all available objects by search params. // objectSearch returns all available objects by search params.
func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]oid.ID, error) { func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]oid.ID, error) {
var filters object.SearchFilters prm := PrmObjectSelect{
filters.AddRootFilter() Container: *p.cid,
ExactAttribute: p.attr,
for _, filter := range p.filters { FilePrefix: p.prefix,
filters.AddFilter(filter.attr, filter.val, object.MatchStringEqual)
} }
if p.prefix != "" { n.prepareAuthParameters(ctx, &prm.PrmAuth)
filters.AddFilter(object.AttributeFileName, p.prefix, object.MatchCommonPrefix)
}
res, err := n.pool.SearchObjects(ctx, *p.cid, filters, n.CallOptions(ctx)...) res, err := n.neoFS.SelectObjects(ctx, prm)
if err != nil {
return nil, fmt.Errorf("init searching using client: %w", err)
}
defer res.Close() return res, n.transformNeofsError(ctx, err)
var num, read int
buf := make([]oid.ID, 10)
for {
num, err = res.Read(buf[read:])
if num > 0 {
read += num
buf = append(buf, oid.ID{})
buf = buf[:cap(buf)]
}
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, n.transformNeofsError(ctx, err)
}
}
return buf[:read], nil
} }
func newAddress(cid *cid.ID, oid *oid.ID) *address.Address { func newAddress(cid *cid.ID, oid *oid.ID) *address.Address {
@ -135,83 +98,69 @@ func newAddress(cid *cid.ID, oid *oid.ID) *address.Address {
// objectHead returns all object's headers. // objectHead returns all object's headers.
func (n *layer) objectHead(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) (*object.Object, error) { func (n *layer) objectHead(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) (*object.Object, error) {
var addr address.Address prm := PrmObjectRead{
Container: *idCnr,
Object: *idObj,
WithHeader: true,
}
addr.SetContainerID(idCnr) n.prepareAuthParameters(ctx, &prm.PrmAuth)
addr.SetObjectID(idObj)
obj, err := n.pool.HeadObject(ctx, addr, n.CallOptions(ctx)...) res, err := n.neoFS.ReadObject(ctx, prm)
return obj, n.transformNeofsError(ctx, err) if err != nil {
return nil, n.transformNeofsError(ctx, err)
}
return res.Head, nil
} }
// writes payload part of the NeoFS object to the provided io.Writer. // writes payload part of the NeoFS object to the provided io.Writer.
// Zero range corresponds to full payload (panics if only offset is set). // Zero range corresponds to full payload (panics if only offset is set).
func (n *layer) objectWritePayload(ctx context.Context, p getParams) error { func (n *layer) objectWritePayload(ctx context.Context, p getParams) error {
// form object address prm := PrmObjectRead{
var a address.Address Container: *p.cid,
Object: *p.oid,
a.SetContainerID(p.cid) WithPayload: true,
a.SetObjectID(p.oid) PayloadRange: [2]uint64{p.off, p.ln},
fmt.Println("objectWritePayload", p.cid, p.oid)
// init payload reader
var r io.ReadCloser
if p.ln+p.off == 0 {
res, err := n.pool.GetObject(ctx, a, n.CallOptions(ctx)...)
if err != nil {
return n.transformNeofsError(ctx, fmt.Errorf("get object using client: %w", err))
}
p.ln = res.Header.PayloadSize()
r = res.Payload
} else {
res, err := n.pool.ObjectRange(ctx, a, p.off, p.ln, n.CallOptions(ctx)...)
if err != nil {
return n.transformNeofsError(ctx, fmt.Errorf("range object payload using client: %w", err))
}
r = res
} }
defer r.Close() n.prepareAuthParameters(ctx, &prm.PrmAuth)
if p.ln > 0 { res, err := n.neoFS.ReadObject(ctx, prm)
if p.ln > 4096 { // configure? if err == nil {
p.ln = 4096 defer res.Payload.Close()
if p.ln == 0 {
p.ln = 4096 // configure?
} }
// alloc buffer for copying // alloc buffer for copying
buf := make([]byte, p.ln) // sync-pool it? buf := make([]byte, p.ln) // sync-pool it?
// copy full payload // copy full payload
_, err := io.CopyBuffer(p.w, r, buf) _, err = io.CopyBuffer(p.w, res.Payload, buf)
if err != nil {
return n.transformNeofsError(ctx, fmt.Errorf("copy payload range: %w", err))
}
} }
return nil return n.transformNeofsError(ctx, err)
} }
// objectGet returns an object with payload in the object. // objectGet returns an object with payload in the object.
func (n *layer) objectGet(ctx context.Context, addr *address.Address) (*object.Object, error) { func (n *layer) objectGet(ctx context.Context, addr *address.Address) (*object.Object, error) {
res, err := n.pool.GetObject(ctx, *addr, n.CallOptions(ctx)...) prm := PrmObjectRead{
Container: *addr.ContainerID(),
Object: *addr.ObjectID(),
WithHeader: true,
WithPayload: true,
}
n.prepareAuthParameters(ctx, &prm.PrmAuth)
res, err := n.neoFS.ReadObject(ctx, prm)
if err != nil { if err != nil {
return nil, n.transformNeofsError(ctx, err) return nil, n.transformNeofsError(ctx, err)
} }
defer res.Payload.Close() return res.Head, nil
payload, err := io.ReadAll(res.Payload)
if err != nil {
return nil, fmt.Errorf("read payload: %w", err)
}
object.NewRawFrom(&res.Header).SetPayload(payload)
return &res.Header, nil
} }
// objectPut into NeoFS, took payload from io.Reader. // objectPut into NeoFS, took payload from io.Reader.
@ -235,9 +184,24 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec
r = d.MultiReader() r = d.MultiReader()
} }
} }
rawObject := formRawObject(p, bkt.CID, own, p.Object)
id, err := n.pool.PutObject(ctx, *rawObject.Object(), r, n.CallOptions(ctx)...) prm := PrmObjectCreate{
Container: *bkt.CID,
Creator: *own,
PayloadSize: uint64(p.Size),
Filename: p.Object,
Payload: r,
}
prm.Attributes = make([][2]string, 0, len(p.Header))
for k, v := range p.Header {
prm.Attributes = append(prm.Attributes, [2]string{k, v})
}
n.prepareAuthParameters(ctx, &prm.PrmAuth)
id, err := n.neoFS.CreateObject(ctx, prm)
if err != nil { if err != nil {
return nil, n.transformNeofsError(ctx, err) return nil, n.transformNeofsError(ctx, err)
} }
@ -292,35 +256,6 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec
}, nil }, nil
} }
func formRawObject(p *PutObjectParams, bktID *cid.ID, own *owner.ID, obj string) *object.RawObject {
attributes := make([]*object.Attribute, 0, len(p.Header)+2)
filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName)
filename.SetValue(obj)
createdAt := object.NewAttribute()
createdAt.SetKey(object.AttributeTimestamp)
createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
attributes = append(attributes, filename, createdAt)
for k, v := range p.Header {
ua := object.NewAttribute()
ua.SetKey(k)
ua.SetValue(v)
attributes = append(attributes, ua)
}
raw := object.NewRaw()
raw.SetOwnerID(own)
raw.SetContainerID(bktID)
raw.SetAttributes(attributes...)
raw.SetPayloadSize(uint64(p.Size))
return raw
}
func updateCRDT2PSetHeaders(header map[string]string, versions *objectVersions, versioningEnabled bool) []*oid.ID { func updateCRDT2PSetHeaders(header map[string]string, versions *objectVersions, versioningEnabled bool) []*oid.ID {
if !versioningEnabled { if !versioningEnabled {
header[versionsUnversionedAttr] = "true" header[versionsUnversionedAttr] = "true"
@ -483,11 +418,17 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
} }
// objectDelete puts tombstone object into neofs. // objectDelete puts tombstone object into neofs.
func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *oid.ID) error { func (n *layer) objectDelete(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) error {
addr := newAddress(cid, oid) prm := PrmObjectDelete{
n.objCache.Delete(addr) Container: *idCnr,
err := n.pool.DeleteObject(ctx, *addr, n.CallOptions(ctx)...) Object: *idObj,
return n.transformNeofsError(ctx, err) }
n.prepareAuthParameters(ctx, &prm.PrmAuth)
n.objCache.Delete(newAddress(idCnr, idObj))
return n.transformNeofsError(ctx, n.neoFS.DeleteObject(ctx, prm))
} }
// ListObjectsV1 returns objects in a bucket for requests of Version 1. // ListObjectsV1 returns objects in a bucket for requests of Version 1.
@ -737,7 +678,7 @@ func (n *layer) transformNeofsError(ctx context.Context, err error) error {
return nil return nil
} }
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { if errors.Is(err, ErrAccessDenied) {
n.log.Debug("error was transformed", zap.String("request_id", api.GetRequestID(ctx)), zap.Error(err)) n.log.Debug("error was transformed", zap.String("request_id", api.GetRequestID(ctx)), zap.Error(err))
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied) return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
} }

53
api/layer/system_object.go
View file

@ -3,8 +3,6 @@ package layer
import ( import (
"context" "context"
"encoding/xml" "encoding/xml"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/errors"
@ -45,8 +43,8 @@ func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objN
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error { func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
f := &findParams{ f := &findParams{
filters: []filter{{attr: objectSystemAttributeName, val: name}}, attr: [2]string{objectSystemAttributeName, name},
cid: bktInfo.CID, cid: bktInfo.CID,
} }
ids, err := n.objectSearch(ctx, f) ids, err := n.objectSearch(ctx, f)
if err != nil { if err != nil {
@ -68,52 +66,41 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) { if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
return nil, err return nil, err
} }
idsToDeleteArr := updateCRDT2PSetHeaders(p.Metadata, versions, false) // false means "last write wins"
attributes := make([]*object.Attribute, 0, 3) prm := PrmObjectCreate{
Container: *p.BktInfo.CID,
Creator: *p.BktInfo.Owner,
Attributes: make([][2]string, 2, 2+len(p.Metadata)),
Payload: p.Reader,
}
filename := object.NewAttribute() prm.Attributes[0][0], prm.Attributes[0][1] = objectSystemAttributeName, p.ObjName
filename.SetKey(objectSystemAttributeName) prm.Attributes[1][0], prm.Attributes[1][1] = attrVersionsIgnore, "true"
filename.SetValue(p.ObjName)
createdAt := object.NewAttribute()
createdAt.SetKey(object.AttributeTimestamp)
createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
versioningIgnore := object.NewAttribute()
versioningIgnore.SetKey(attrVersionsIgnore)
versioningIgnore.SetValue(strconv.FormatBool(true))
attributes = append(attributes, filename, createdAt, versioningIgnore)
for k, v := range p.Metadata { for k, v := range p.Metadata {
attr := object.NewAttribute()
if !IsSystemHeader(k) { if !IsSystemHeader(k) {
k = p.Prefix + k k = p.Prefix + k
} }
attr.SetKey(k)
if p.Prefix == tagPrefix && v == "" { if v == "" && p.Prefix == tagPrefix {
v = tagEmptyMark v = tagEmptyMark
} }
attr.SetValue(v)
attributes = append(attributes, attr) prm.Attributes = append(prm.Attributes, [2]string{k, v})
} }
raw := object.NewRaw() id, err := n.neoFS.CreateObject(ctx, prm)
raw.SetOwnerID(p.BktInfo.Owner)
raw.SetContainerID(p.BktInfo.CID)
raw.SetAttributes(attributes...)
oid, err := n.pool.PutObject(ctx, *raw.Object(), p.Reader, n.CallOptions(ctx)...)
if err != nil { if err != nil {
return nil, n.transformNeofsError(ctx, err) return nil, n.transformNeofsError(ctx, err)
} }
meta, err := n.objectHead(ctx, p.BktInfo.CID, oid) meta, err := n.objectHead(ctx, p.BktInfo.CID, id)
if err != nil { if err != nil {
return nil, err return nil, err
} }
idsToDeleteArr := updateCRDT2PSetHeaders(p.Metadata, versions, false) // false means "last write wins"
for _, id := range idsToDeleteArr { for _, id := range idsToDeleteArr {
if err = n.objectDelete(ctx, p.BktInfo.CID, id); err != nil { if err = n.objectDelete(ctx, p.BktInfo.CID, id); err != nil {
n.log.Warn("couldn't delete system object", n.log.Warn("couldn't delete system object",
@ -178,8 +165,8 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName strin
func (n *layer) headSystemVersions(ctx context.Context, bkt *data.BucketInfo, sysName string) (*objectVersions, error) { func (n *layer) headSystemVersions(ctx context.Context, bkt *data.BucketInfo, sysName string) (*objectVersions, error) {
f := &findParams{ f := &findParams{
filters: []filter{{attr: objectSystemAttributeName, val: sysName}}, attr: [2]string{objectSystemAttributeName, sysName},
cid: bkt.CID, cid: bkt.CID,
} }
ids, err := n.objectSearch(ctx, f) ids, err := n.objectSearch(ctx, f)
if err != nil { if err != nil {

343
api/layer/versioning_test.go
View file

@ -15,47 +15,168 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/accounting"
"github.com/nspcc-dev/neofs-sdk-go/container" "github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/nspcc-dev/neofs-sdk-go/logger"
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/object/id/test" "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/pool" tokentest "github.com/nspcc-dev/neofs-sdk-go/token/test"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/token"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
type testPool struct { type testNeoFS struct {
pool.Pool NeoFS
objects map[string]*object.Object objects map[string]*object.Object
containers map[string]*container.Container containers map[string]*container.Container
currentEpoch uint64 currentEpoch uint64
} }
func newTestPool() *testPool { func (t *testNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (*cid.ID, error) {
return &testPool{ var opts []container.Option
objects: make(map[string]*object.Object),
containers: make(map[string]*container.Container), opts = append(opts,
container.WithOwnerID(&prm.Creator),
container.WithPolicy(&prm.Policy),
container.WithCustomBasicACL(prm.BasicACL),
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(prm.Time.Unix(), 10)),
)
if prm.Name != "" {
opts = append(opts, container.WithAttribute(container.AttributeName, prm.Name))
} }
cnr := container.New(opts...)
cnr.SetSessionToken(prm.SessionToken)
if prm.Name != "" {
container.SetNativeName(cnr, prm.Name)
}
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return nil, err
}
id := cid.New()
id.SetSHA256(sha256.Sum256(b))
t.containers[id.String()] = cnr
return id, nil
} }
func (t *testPool) PutObject(_ context.Context, hdr object.Object, payload io.Reader, _ ...pool.CallOption) (*oid.ID, error) { func (t *testNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
for k, v := range t.containers {
if k == id.String() {
return v, nil
}
}
return nil, fmt.Errorf("container not found " + id.String())
}
func (t *testNeoFS) UserContainers(_ context.Context, _ owner.ID) ([]cid.ID, error) {
var res []cid.ID
for k := range t.containers {
var idCnr cid.ID
if err := idCnr.Parse(k); err != nil {
return nil, err
}
res = append(res, idCnr)
}
return res, nil
}
func (t *testNeoFS) SelectObjects(_ context.Context, prm PrmObjectSelect) ([]oid.ID, error) {
var filters object.SearchFilters
filters.AddRootFilter()
if prm.FilePrefix != "" {
filters.AddFilter(object.AttributeFileName, prm.FilePrefix, object.MatchCommonPrefix)
}
if prm.ExactAttribute[0] != "" {
filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual)
}
cidStr := prm.Container.String()
var res []oid.ID
if len(filters) == 1 {
for k, v := range t.objects {
if strings.Contains(k, cidStr) {
res = append(res, *v.ID())
}
}
return res, nil
}
filter := filters[1]
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual ||
(filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) {
return nil, fmt.Errorf("usupported filters")
}
for k, v := range t.objects {
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
res = append(res, *v.ID())
}
}
return res, nil
}
func (t *testNeoFS) ReadObject(_ context.Context, prm PrmObjectRead) (*ObjectPart, error) {
var addr address.Address
addr.SetContainerID(&prm.Container)
addr.SetObjectID(&prm.Object)
sAddr := addr.String()
if obj, ok := t.objects[sAddr]; ok {
return &ObjectPart{
Head: obj,
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
}, nil
}
return nil, fmt.Errorf("object not found " + addr.String())
}
func (t *testNeoFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*oid.ID, error) {
id := test.ID() id := test.ID()
raw := object.NewRawFrom(&hdr) attrs := make([]*object.Attribute, 0)
if prm.Filename != "" {
a := object.NewAttribute()
a.SetKey(object.AttributeFileName)
a.SetValue(prm.Filename)
attrs = append(attrs, a)
}
for i := range prm.Attributes {
a := object.NewAttribute()
a.SetKey(prm.Attributes[i][0])
a.SetValue(prm.Attributes[i][1])
attrs = append(attrs, a)
}
raw := object.NewRaw()
raw.SetContainerID(&prm.Container)
raw.SetID(id) raw.SetID(id)
raw.SetPayloadSize(prm.PayloadSize)
raw.SetAttributes(attrs...)
raw.SetCreationEpoch(t.currentEpoch) raw.SetCreationEpoch(t.currentEpoch)
t.currentEpoch++ t.currentEpoch++
if payload != nil { if prm.Payload != nil {
all, err := io.ReadAll(payload) all, err := io.ReadAll(prm.Payload)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -67,60 +188,21 @@ func (t *testPool) PutObject(_ context.Context, hdr object.Object, payload io.Re
return raw.ID(), nil return raw.ID(), nil
} }
func (t *testPool) DeleteObject(ctx context.Context, addr address.Address, option ...pool.CallOption) error { func (t *testNeoFS) DeleteObject(_ context.Context, prm PrmObjectDelete) error {
var addr address.Address
addr.SetContainerID(&prm.Container)
addr.SetObjectID(&prm.Object)
delete(t.objects, addr.String()) delete(t.objects, addr.String())
return nil return nil
} }
func (t *testPool) GetObject(_ context.Context, addr address.Address, _ ...pool.CallOption) (*pool.ResGetObject, error) { func newTestPool() *testNeoFS {
sAddr := addr.String() return &testNeoFS{
objects: make(map[string]*object.Object),
if obj, ok := t.objects[sAddr]; ok { containers: make(map[string]*container.Container),
return &pool.ResGetObject{
Header: *obj,
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
}, nil
} }
return nil, fmt.Errorf("object not found " + addr.String())
}
func (t *testPool) HeadObject(ctx context.Context, addr address.Address, _ ...pool.CallOption) (*object.Object, error) {
res, err := t.GetObject(ctx, addr)
if err != nil {
return nil, err
}
return &res.Header, nil
}
func (t *testPool) SearchObjects(_ context.Context, idCnr cid.ID, filters object.SearchFilters, _ ...pool.CallOption) (*pool.ResObjectSearch, error) {
cidStr := idCnr.String()
var res []*oid.ID
if len(filters) == 1 {
for k, v := range t.objects {
if strings.Contains(k, cidStr) {
res = append(res, v.ID())
}
}
return nil, nil
}
filter := filters[1]
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual ||
(filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) {
return nil, fmt.Errorf("usupported filters")
}
for k, v := range t.objects {
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
res = append(res, v.ID())
}
}
return nil, nil
} }
func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool { func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool {
@ -133,79 +215,6 @@ func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool
return false return false
} }
func (t *testPool) PutContainer(ctx context.Context, container *container.Container, option ...pool.CallOption) (*cid.ID, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return nil, err
}
id := cid.New()
id.SetSHA256(sha256.Sum256(b))
t.containers[id.String()] = container
return id, nil
}
func (t *testPool) GetContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*container.Container, error) {
for k, v := range t.containers {
if k == id.String() {
return v, nil
}
}
return nil, fmt.Errorf("container not found " + id.String())
}
func (t *testPool) ListContainers(ctx context.Context, id *owner.ID, option ...pool.CallOption) ([]*cid.ID, error) {
var res []*cid.ID
for k := range t.containers {
cID := cid.New()
if err := cID.Parse(k); err != nil {
return nil, err
}
res = append(res, cID)
}
return res, nil
}
func (t *testPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) error {
delete(t.containers, id.String())
return nil
}
func (t *testPool) GetEACL(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*eacl.Table, error) {
panic("implement me")
}
func (t *testPool) Balance(ctx context.Context, owner *owner.ID, opts ...pool.CallOption) (*accounting.Decimal, error) {
panic("implement me")
}
func (t *testPool) SetEACL(ctx context.Context, table *eacl.Table, option ...pool.CallOption) error {
panic("implement me")
}
func (t *testPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...pool.CallOption) error {
panic("implement me")
}
func (t *testPool) Connection() (pool.Client, *session.Token, error) {
panic("implement me")
}
func (t *testPool) Close() {
panic("implement me")
}
func (t *testPool) OwnerID() *owner.ID {
return nil
}
func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error {
return nil
}
func (tc *testContext) putObject(content []byte) *data.ObjectInfo { func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{ objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
Bucket: tc.bktID.String(), Bucket: tc.bktID.String(),
@ -298,7 +307,7 @@ func (tc *testContext) checkListObjects(ids ...*oid.ID) {
} }
func (tc *testContext) getSystemObject(objectName string) *object.Object { func (tc *testContext) getSystemObject(objectName string) *object.Object {
for _, obj := range tc.testPool.objects { for _, obj := range tc.testNeoFS.objects {
for _, attr := range obj.Attributes() { for _, attr := range obj.Attributes() {
if attr.Key() == objectSystemAttributeName && attr.Value() == objectName { if attr.Key() == objectSystemAttributeName && attr.Value() == objectName {
return obj return obj
@ -309,22 +318,25 @@ func (tc *testContext) getSystemObject(objectName string) *object.Object {
} }
type testContext struct { type testContext struct {
t *testing.T t *testing.T
ctx context.Context ctx context.Context
layer Client layer Client
bkt string bkt string
bktID *cid.ID bktID *cid.ID
obj string obj string
testPool *testPool testNeoFS *testNeoFS
} }
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
key, err := keys.NewPrivateKey() key, err := keys.NewPrivateKey()
require.NoError(t, err) require.NoError(t, err)
bearerToken := tokentest.BearerToken()
require.NoError(t, bearerToken.SignToken(&key.PrivateKey))
ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{ ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{
Gate: &accessbox.GateData{ Gate: &accessbox.GateData{
BearerToken: token.NewBearerToken(), BearerToken: bearerToken,
GateKey: key.PublicKey(), GateKey: key.PublicKey(),
}, },
}) })
@ -333,8 +345,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
tp := newTestPool() tp := newTestPool()
bktName := "testbucket1" bktName := "testbucket1"
cnr := container.New(container.WithAttribute(container.AttributeName, bktName)) bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
bktID, err := tp.PutContainer(ctx, cnr) Name: bktName,
})
require.NoError(t, err) require.NoError(t, err)
config := DefaultCachesConfigs() config := DefaultCachesConfigs()
@ -348,19 +361,17 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
} }
return &testContext{ return &testContext{
ctx: ctx, ctx: ctx,
layer: NewLayer(l, tp, layerCfg), layer: NewLayer(l, tp, layerCfg),
bkt: bktName, bkt: bktName,
bktID: bktID, bktID: bktID,
obj: "obj1", obj: "obj1",
t: t, t: t,
testPool: tp, testNeoFS: tp,
} }
} }
func TestSimpleVersioning(t *testing.T) { func TestSimpleVersioning(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
tc := prepareContext(t) tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bktID.String(), Bucket: tc.bktID.String(),
@ -385,8 +396,6 @@ func TestSimpleVersioning(t *testing.T) {
} }
func TestSimpleNoVersioning(t *testing.T) { func TestSimpleNoVersioning(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
tc := prepareContext(t) tc := prepareContext(t)
obj1Content1 := []byte("content obj1 v1") obj1Content1 := []byte("content obj1 v1")
@ -404,8 +413,6 @@ func TestSimpleNoVersioning(t *testing.T) {
} }
func TestVersioningDeleteObject(t *testing.T) { func TestVersioningDeleteObject(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
tc := prepareContext(t) tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bktID.String(), Bucket: tc.bktID.String(),
@ -423,8 +430,6 @@ func TestVersioningDeleteObject(t *testing.T) {
} }
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) { func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
tc := prepareContext(t) tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bktID.String(), Bucket: tc.bktID.String(),
@ -532,8 +537,6 @@ func TestGetLastVersion(t *testing.T) {
} }
func TestNoVersioningDeleteObject(t *testing.T) { func TestNoVersioningDeleteObject(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
tc := prepareContext(t) tc := prepareContext(t)
tc.putObject([]byte("content obj1 v1")) tc.putObject([]byte("content obj1 v1"))
@ -789,8 +792,6 @@ func TestUpdateCRDT2PSetHeaders(t *testing.T) {
} }
func TestSystemObjectsVersioning(t *testing.T) { func TestSystemObjectsVersioning(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
cacheConfig := DefaultCachesConfigs() cacheConfig := DefaultCachesConfigs()
cacheConfig.System.Lifetime = 0 cacheConfig.System.Lifetime = 0
@ -801,7 +802,7 @@ func TestSystemObjectsVersioning(t *testing.T) {
}) })
require.NoError(t, err) require.NoError(t, err)
objMeta, ok := tc.testPool.objects[objInfo.Address().String()] objMeta, ok := tc.testNeoFS.objects[objInfo.Address().String()]
require.True(t, ok) require.True(t, ok)
_, err = tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ _, err = tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
@ -811,7 +812,7 @@ func TestSystemObjectsVersioning(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// simulate failed deletion // simulate failed deletion
tc.testPool.objects[objInfo.Address().String()] = objMeta tc.testNeoFS.objects[objInfo.Address().String()] = objMeta
versioning, err := tc.layer.GetBucketVersioning(tc.ctx, tc.bkt) versioning, err := tc.layer.GetBucketVersioning(tc.ctx, tc.bkt)
require.NoError(t, err) require.NoError(t, err)
@ -819,8 +820,6 @@ func TestSystemObjectsVersioning(t *testing.T) {
} }
func TestDeleteSystemObjectsVersioning(t *testing.T) { func TestDeleteSystemObjectsVersioning(t *testing.T) {
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
t.Skip("pool.Pool does not support overriding")
cacheConfig := DefaultCachesConfigs() cacheConfig := DefaultCachesConfigs()
cacheConfig.System.Lifetime = 0 cacheConfig.System.Lifetime = 0
@ -840,7 +839,7 @@ func TestDeleteSystemObjectsVersioning(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// simulate failed deletion // simulate failed deletion
tc.testPool.objects[newAddress(objMeta.ContainerID(), objMeta.ID()).String()] = objMeta tc.testNeoFS.objects[newAddress(objMeta.ContainerID(), objMeta.ID()).String()] = objMeta
tagging, err := tc.layer.GetBucketTagging(tc.ctx, tc.bkt) tagging, err := tc.layer.GetBucketTagging(tc.ctx, tc.bkt)
require.NoError(t, err) require.NoError(t, err)

52
api/resolver/resolver.go
View file

@ -5,24 +5,27 @@ import (
"fmt" "fmt"
"github.com/nspcc-dev/neo-go/pkg/rpc/client" "github.com/nspcc-dev/neo-go/pkg/rpc/client"
neofsclient "github.com/nspcc-dev/neofs-sdk-go/client"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/resolver" "github.com/nspcc-dev/neofs-sdk-go/resolver"
) )
const ( const (
NNSResolver = "nns" NNSResolver = "nns"
DNSResolver = "dns" DNSResolver = "dns"
networkSystemDNSParam = "SystemDNS"
) )
// NeoFS represents virtual connection to the NeoFS network.
type NeoFS interface {
// SystemDNS reads system DNS network parameters of the NeoFS.
//
// Returns exactly on non-zero value. Returns any error encountered
// which prevented the parameter to be read.
SystemDNS(context.Context) (string, error)
}
type Config struct { type Config struct {
Pool pool.Pool NeoFS NeoFS
RPC *client.Client RPC *client.Client
} }
type BucketResolver struct { type BucketResolver struct {
@ -69,7 +72,7 @@ func NewResolver(order []string, cfg *Config) (*BucketResolver, error) {
func newResolver(name string, cfg *Config, next *BucketResolver) (*BucketResolver, error) { func newResolver(name string, cfg *Config, next *BucketResolver) (*BucketResolver, error) {
switch name { switch name {
case DNSResolver: case DNSResolver:
return NewDNSResolver(cfg.Pool, next) return NewDNSResolver(cfg.NeoFS, next)
case NNSResolver: case NNSResolver:
return NewNNSResolver(cfg.RPC, next) return NewNNSResolver(cfg.RPC, next)
default: default:
@ -77,38 +80,15 @@ func newResolver(name string, cfg *Config, next *BucketResolver) (*BucketResolve
} }
} }
func NewDNSResolver(p pool.Pool, next *BucketResolver) (*BucketResolver, error) { func NewDNSResolver(neoFS NeoFS, next *BucketResolver) (*BucketResolver, error) {
if p == nil { if neoFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver") return nil, fmt.Errorf("pool must not be nil for DNS resolver")
} }
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
conn, _, err := p.Connection() domain, err := neoFS.SystemDNS(ctx)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
}
networkInfoRes, err := conn.NetworkInfo(ctx, neofsclient.PrmNetworkInfo{})
if err == nil {
err = apistatus.ErrFromStatus(networkInfoRes.Status())
}
if err != nil {
return nil, err
}
networkInfo := networkInfoRes.Info()
var domain string
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == networkSystemDNSParam {
domain = string(parameter.Value())
return true
}
return false
})
if domain == "" {
return nil, fmt.Errorf("couldn't resolve container '%s': not found", name)
} }
domain = name + "." + domain domain = name + "." + domain

139
authmate/authmate.go
View file

@ -3,14 +3,12 @@ package authmate
import ( import (
"context" "context"
"crypto/ecdsa" "crypto/ecdsa"
"encoding/binary"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"math" "math"
"os" "os"
"strconv"
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
@ -18,35 +16,75 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/acl"
"github.com/nspcc-dev/neofs-sdk-go/client"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/object/address"
"github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/token" "github.com/nspcc-dev/neofs-sdk-go/token"
"go.uber.org/zap" "go.uber.org/zap"
) )
const ( // PrmContainerCreate groups parameters of containers created by authmate.
defaultAuthContainerBasicACL acl.BasicACL = 0b00111100100011001000110011001110 // 0x3C8C8CCE - private container with only GET allowed to others type PrmContainerCreate struct {
) // NeoFS identifier of the container creator.
Owner owner.ID
// Container placement policy.
Policy netmap.PlacementPolicy
// Friendly name for the container (optional).
FriendlyName string
}
// NetworkState represents NeoFS network state which is needed for authmate processing.
type NetworkState struct {
// Current NeoFS time.
Epoch uint64
// Duration of the Morph chain block in ms.
BlockDuration int64
// Duration of the NeoFS epoch in Morph chain blocks.
EpochDuration uint64
}
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// NeoFS interface required by credential tool.
tokens.NeoFS
// ContainerExists checks container presence in NeoFS by identifier.
// Returns nil iff container exists.
ContainerExists(context.Context, cid.ID) error
// CreateContainer creates and saves parameterized container in NeoFS.
// Returns ID of the saved container.
//
// The container must be private with GET access of OTHERS group.
// Creation time should also be stamped.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the container to be created.
CreateContainer(context.Context, PrmContainerCreate) (*cid.ID, error)
// NetworkState returns current state of the NeoFS network.
// Returns any error encountered which prevented state to be read.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the state to be read.
NetworkState(context.Context) (*NetworkState, error)
}
// Agent contains client communicating with NeoFS and logger. // Agent contains client communicating with NeoFS and logger.
type Agent struct { type Agent struct {
pool pool.Pool neoFS NeoFS
log *zap.Logger log *zap.Logger
} }
// New creates an object of type Agent that consists of Client and logger. // New creates an object of type Agent that consists of Client and logger.
func New(log *zap.Logger, conns pool.Pool) *Agent { func New(log *zap.Logger, neoFS NeoFS) *Agent {
return &Agent{log: log, pool: conns} return &Agent{log: log, neoFS: neoFS}
} }
type ( type (
@ -85,12 +123,6 @@ type lifetimeOptions struct {
Exp uint64 Exp uint64
} }
type epochDurations struct {
currentEpoch uint64
msPerBlock int64
blocksInEpoch uint64
}
type ( type (
issuingResult struct { issuingResult struct {
AccessKeyID string `json:"access_key_id"` AccessKeyID string `json:"access_key_id"`
@ -108,8 +140,7 @@ type (
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner *owner.ID) (*cid.ID, error) { func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner *owner.ID) (*cid.ID, error) {
if opts.ID != nil { if opts.ID != nil {
// check that container exists // check that container exists
_, err := a.pool.GetContainer(ctx, opts.ID) return opts.ID, a.neoFS.ContainerExists(ctx, *opts.ID)
return opts.ID, err
} }
pp, err := policy.Parse(opts.PlacementPolicy) pp, err := policy.Parse(opts.PlacementPolicy)
@ -117,62 +148,18 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
return nil, fmt.Errorf("failed to build placement policy: %w", err) return nil, fmt.Errorf("failed to build placement policy: %w", err)
} }
cnrOptions := []container.Option{ cnrID, err := a.neoFS.CreateContainer(ctx, PrmContainerCreate{
container.WithPolicy(pp), Owner: *idOwner,
container.WithCustomBasicACL(defaultAuthContainerBasicACL), Policy: *pp,
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)), FriendlyName: opts.FriendlyName,
container.WithOwnerID(idOwner), })
}
if opts.FriendlyName != "" {
cnrOptions = append(cnrOptions, container.WithAttribute(container.AttributeName, opts.FriendlyName))
}
cnr := container.New(cnrOptions...)
if opts.FriendlyName != "" {
container.SetNativeName(cnr, opts.FriendlyName)
}
cnrID, err := a.pool.PutContainer(ctx, cnr)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := a.pool.WaitForContainerPresence(ctx, cnrID, pool.DefaultPollingParams()); err != nil {
return nil, err
}
return cnrID, nil return cnrID, nil
} }
func (a *Agent) getEpochDurations(ctx context.Context) (*epochDurations, error) {
if conn, _, err := a.pool.Connection(); err != nil {
return nil, err
} else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil {
return nil, err
} else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil {
return nil, err
} else {
networkInfo := networkInfoRes.Info()
res := &epochDurations{
currentEpoch: networkInfo.CurrentEpoch(),
msPerBlock: networkInfo.MsPerBlock(),
}
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "EpochDuration" {
data := make([]byte, 8)
copy(data, parameter.Value())
res.blocksInEpoch = binary.LittleEndian.Uint64(data)
return true
}
return false
})
if res.blocksInEpoch == 0 {
return nil, fmt.Errorf("not found param: EpochDuration")
}
return res, nil
}
}
func checkPolicy(policyString string) (*netmap.PlacementPolicy, error) { func checkPolicy(policyString string) (*netmap.PlacementPolicy, error) {
result, err := policy.Parse(policyString) result, err := policy.Parse(policyString)
if err == nil { if err == nil {
@ -226,12 +213,12 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return err return err
} }
durations, err := a.getEpochDurations(ctx) netState, err := a.neoFS.NetworkState(ctx)
if err != nil { if err != nil {
return err return err
} }
lifetime.Iat = durations.currentEpoch lifetime.Iat = netState.Epoch
msPerEpoch := durations.blocksInEpoch * uint64(durations.msPerBlock) msPerEpoch := netState.EpochDuration * uint64(netState.BlockDuration)
epochLifetime := uint64(options.Lifetime.Milliseconds()) / msPerEpoch epochLifetime := uint64(options.Lifetime.Milliseconds()) / msPerEpoch
if uint64(options.Lifetime.Milliseconds())%msPerEpoch != 0 { if uint64(options.Lifetime.Milliseconds())%msPerEpoch != 0 {
epochLifetime++ epochLifetime++
@ -268,7 +255,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
zap.Stringer("owner_tkn", idOwner)) zap.Stringer("owner_tkn", idOwner))
addr, err := tokens. addr, err := tokens.
New(a.pool, secrets.EphemeralKey, cache.DefaultAccessBoxConfig()). New(a.neoFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig()).
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...) Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
if err != nil { if err != nil {
return fmt.Errorf("failed to put bearer token: %w", err) return fmt.Errorf("failed to put bearer token: %w", err)
@ -310,7 +297,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
// ObtainSecret receives an existing secret access key from NeoFS and // ObtainSecret receives an existing secret access key from NeoFS and
// writes to io.Writer the secret access key. // writes to io.Writer the secret access key.
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error { func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
bearerCreds := tokens.New(a.pool, options.GatePrivateKey, cache.DefaultAccessBoxConfig()) bearerCreds := tokens.New(a.neoFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig())
addr := address.NewAddress() addr := address.NewAddress()
if err := addr.Parse(options.SecretAddress); err != nil { if err := addr.Parse(options.SecretAddress); err != nil {
return fmt.Errorf("failed to parse secret address: %w", err) return fmt.Errorf("failed to parse secret address: %w", err)

25
cmd/authmate/main.go
View file

@ -13,6 +13,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/authmate" "github.com/nspcc-dev/neofs-s3-gw/authmate"
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
"github.com/nspcc-dev/neofs-s3-gw/internal/version" "github.com/nspcc-dev/neofs-s3-gw/internal/version"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet" "github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
@ -238,12 +239,12 @@ It will be ceil rounded to the nearest amount of epoch.`,
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
client, err := createSDKClient(ctx, log, &key.PrivateKey, peerAddressFlag) neoFS, err := createNeoFS(ctx, log, &key.PrivateKey, peerAddressFlag)
if err != nil { if err != nil {
return cli.Exit(fmt.Sprintf("failed to create sdk client: %s", err), 2) return cli.Exit(fmt.Sprintf("failed to create NeoFS component: %s", err), 2)
} }
agent := authmate.New(log, client) agent := authmate.New(log, neoFS)
var containerID *cid.ID var containerID *cid.ID
if len(containerIDFlag) > 0 { if len(containerIDFlag) > 0 {
containerID = cid.New() containerID = cid.New()
@ -388,12 +389,12 @@ func obtainSecret() *cli.Command {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
client, err := createSDKClient(ctx, log, &key.PrivateKey, peerAddressFlag) neoFS, err := createNeoFS(ctx, log, &key.PrivateKey, peerAddressFlag)
if err != nil { if err != nil {
return cli.Exit(fmt.Sprintf("failed to create sdk client: %s", err), 2) return cli.Exit(fmt.Sprintf("failed to create NeoFS component: %s", err), 2)
} }
agent := authmate.New(log, client) agent := authmate.New(log, neoFS)
var _ = agent var _ = agent
@ -424,7 +425,7 @@ func obtainSecret() *cli.Command {
return command return command
} }
func createSDKClient(ctx context.Context, log *zap.Logger, key *ecdsa.PrivateKey, peerAddress string) (pool.Pool, error) { func createNeoFS(ctx context.Context, log *zap.Logger, key *ecdsa.PrivateKey, peerAddress string) (authmate.NeoFS, error) {
log.Debug("prepare connection pool") log.Debug("prepare connection pool")
pb := new(pool.Builder) pb := new(pool.Builder)
@ -435,5 +436,13 @@ func createSDKClient(ctx context.Context, log *zap.Logger, key *ecdsa.PrivateKey
NodeConnectionTimeout: poolConnectTimeout, NodeConnectionTimeout: poolConnectTimeout,
NodeRequestTimeout: poolRequestTimeout, NodeRequestTimeout: poolRequestTimeout,
} }
return pb.Build(ctx, opts) p, err := pb.Build(ctx, opts)
if err != nil {
return nil, err
}
var neoFS neofs.AuthmateNeoFS
neoFS.SetConnectionPool(p)
return &neoFS, nil
} }

47
cmd/s3-gw/app.go
View file

@ -17,6 +17,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/notifications" "github.com/nspcc-dev/neofs-s3-gw/api/notifications"
"github.com/nspcc-dev/neofs-s3-gw/api/resolver" "github.com/nspcc-dev/neofs-s3-gw/api/resolver"
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
"github.com/nspcc-dev/neofs-s3-gw/internal/version" "github.com/nspcc-dev/neofs-s3-gw/internal/version"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet" "github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
"github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/policy"
@ -28,13 +29,13 @@ import (
type ( type (
// App is the main application structure. // App is the main application structure.
App struct { App struct {
pool pool.Pool ctr auth.Center
ctr auth.Center log *zap.Logger
log *zap.Logger cfg *viper.Viper
cfg *viper.Viper tls *tlsConfig
tls *tlsConfig obj layer.Client
obj layer.Client api api.Handler
api api.Handler nc *notifications.Controller
maxClients api.MaxClients maxClients api.MaxClients
@ -50,7 +51,6 @@ type (
func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App { func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
var ( var (
conns pool.Pool
key *keys.PrivateKey key *keys.PrivateKey
err error err error
tls *tlsConfig tls *tlsConfig
@ -109,7 +109,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
NodeRequestTimeout: reqTimeout, NodeRequestTimeout: reqTimeout,
ClientRebalanceInterval: reBalance, ClientRebalanceInterval: reBalance,
} }
conns, err = poolPeers.Build(ctx, opts) conns, err := poolPeers.Build(ctx, opts)
if err != nil { if err != nil {
l.Fatal("failed to create connection pool", zap.Error(err)) l.Fatal("failed to create connection pool", zap.Error(err))
} }
@ -120,8 +120,11 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
l.Fatal("couldn't generate random key", zap.Error(err)) l.Fatal("couldn't generate random key", zap.Error(err))
} }
var neoFS neofs.NeoFS
neoFS.SetConnectionPool(conns)
resolveCfg := &resolver.Config{ resolveCfg := &resolver.Config{
Pool: conns, NeoFS: &neoFS,
} }
if rpcEndpoint := v.GetString(cfgRPCEndpoint); rpcEndpoint != "" { if rpcEndpoint := v.GetString(cfgRPCEndpoint); rpcEndpoint != "" {
@ -155,12 +158,16 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
NotificationController: nc, NotificationController: nc,
} }
var n neofs.NeoFS
n.SetConnectionPool(conns)
// prepare object layer // prepare object layer
obj = layer.NewLayer(l, conns, layerCfg) obj = layer.NewLayer(l, &layerNeoFS{&n}, layerCfg)
// prepare auth center // prepare auth center
ctr = auth.New(conns, key, getAccessBoxCacheConfig(v, l)) ctr = auth.New(&neofs.AuthmateNeoFS{
NeoFS: n,
}, key, getAccessBoxCacheConfig(v, l))
handlerOptions := getHandlerOptions(v, l) handlerOptions := getHandlerOptions(v, l)
if caller, err = handler.New(l, obj, handlerOptions); err != nil { if caller, err = handler.New(l, obj, handlerOptions); err != nil {
@ -168,13 +175,13 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
} }
return &App{ return &App{
ctr: ctr, ctr: ctr,
pool: conns, log: l,
log: l, cfg: v,
cfg: v, obj: obj,
obj: obj, tls: tls,
tls: tls, api: caller,
api: caller, nc: nc,
webDone: make(chan struct{}, 1), webDone: make(chan struct{}, 1),
wrkDone: make(chan struct{}, 1), wrkDone: make(chan struct{}, 1),

42
cmd/s3-gw/neofs.go Normal file
View file

@ -0,0 +1,42 @@
package main
import (
"context"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// mediator which implements layer.NeoFS through neofs.NeoFS.
type layerNeoFS struct {
*neofs.NeoFS
}
func (x *layerNeoFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreate) (*cid.ID, error) {
return x.NeoFS.CreateContainer(ctx, neofs.PrmContainerCreate{
Creator: prm.Creator,
Policy: prm.Policy,
Name: prm.Name,
Time: prm.Time,
BasicACL: prm.BasicACL,
SessionToken: prm.SessionToken,
LocationConstraintAttribute: prm.LocationConstraintAttribute,
})
}
func (x *layerNeoFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (*oid.ID, error) {
return x.NeoFS.CreateObject(ctx, neofs.PrmObjectCreate{
Creator: prm.Creator,
Container: prm.Container,
Time: time.Now().UTC(),
Filename: prm.Filename,
PayloadSize: prm.PayloadSize,
Attributes: prm.Attributes,
Payload: prm.Payload,
BearerToken: prm.BearerToken,
PrivateKey: prm.PrivateKey,
})
}

102
creds/tokens/credentials.go
View file

@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"strconv" "strconv"
"time" "time"
@ -12,10 +11,9 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/pool"
) )
type ( type (
@ -27,11 +25,49 @@ type (
cred struct { cred struct {
key *keys.PrivateKey key *keys.PrivateKey
pool pool.Pool neoFS NeoFS
cache *cache.AccessBoxCache cache *cache.AccessBoxCache
} }
) )
// PrmObjectCreate groups parameters of objects created by credential tool.
type PrmObjectCreate struct {
// NeoFS identifier of the object creator.
Creator owner.ID
// NeoFS container to store the object.
Container cid.ID
// Object creation time.
Time time.Time
// File name.
Filename string
// Last NeoFS epoch of the object lifetime.
ExpirationEpoch uint64
// Object payload.
Payload []byte
}
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// CreateObject creates and saves a parameterized object in the specified
// NeoFS container from a specific user. Returns ID of the saved object.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the object to be created.
CreateObject(context.Context, PrmObjectCreate) (*oid.ID, error)
// ReadObjectPayload reads payload of the object from NeoFS network by address
// into memory.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the object payload to be read.
ReadObjectPayload(context.Context, address.Address) ([]byte, error)
}
var ( var (
// ErrEmptyPublicKeys is returned when no HCS keys are provided. // ErrEmptyPublicKeys is returned when no HCS keys are provided.
ErrEmptyPublicKeys = errors.New("HCS public keys could not be empty") ErrEmptyPublicKeys = errors.New("HCS public keys could not be empty")
@ -42,8 +78,8 @@ var (
var _ = New var _ = New
// New creates new Credentials instance using given cli and key. // New creates new Credentials instance using given cli and key.
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Credentials { func New(neoFS NeoFS, key *keys.PrivateKey, config *cache.Config) Credentials {
return &cred{pool: conns, key: key, cache: cache.NewAccessBoxCache(config)} return &cred{neoFS: neoFS, key: key, cache: cache.NewAccessBoxCache(config)}
} }
func (c *cred) GetBox(ctx context.Context, addr *address.Address) (*accessbox.Box, error) { func (c *cred) GetBox(ctx context.Context, addr *address.Address) (*accessbox.Box, error) {
@ -70,24 +106,9 @@ func (c *cred) GetBox(ctx context.Context, addr *address.Address) (*accessbox.Bo
} }
func (c *cred) getAccessBox(ctx context.Context, addr *address.Address) (*accessbox.AccessBox, error) { func (c *cred) getAccessBox(ctx context.Context, addr *address.Address) (*accessbox.AccessBox, error) {
// init payload reader data, err := c.neoFS.ReadObjectPayload(ctx, *addr)
res, err := c.pool.GetObject(ctx, *addr)
if err != nil { if err != nil {
return nil, fmt.Errorf("client pool failure: %w", err) return nil, fmt.Errorf("read payload: %w", err)
}
defer res.Payload.Close()
// read payload
var data []byte
if sz := res.Header.PayloadSize(); sz > 0 {
data = make([]byte, sz)
_, err = io.ReadFull(res.Payload, data)
if err != nil {
return nil, fmt.Errorf("read payload: %w", err)
}
} }
// decode access box // decode access box
@ -99,10 +120,10 @@ func (c *cred) getAccessBox(ctx context.Context, addr *address.Address) (*access
return &box, nil return &box, nil
} }
func (c *cred) Put(ctx context.Context, cid *cid.ID, issuer *owner.ID, box *accessbox.AccessBox, expiration uint64, keys ...*keys.PublicKey) (*address.Address, error) { func (c *cred) Put(ctx context.Context, idCnr *cid.ID, issuer *owner.ID, box *accessbox.AccessBox, expiration uint64, keys ...*keys.PublicKey) (*address.Address, error) {
var ( var (
err error err error
created = strconv.FormatInt(time.Now().Unix(), 10) created = time.Now()
) )
if len(keys) == 0 { if len(keys) == 0 {
@ -115,31 +136,20 @@ func (c *cred) Put(ctx context.Context, cid *cid.ID, issuer *owner.ID, box *acce
return nil, err return nil, err
} }
timestamp := object.NewAttribute() idObj, err := c.neoFS.CreateObject(ctx, PrmObjectCreate{
timestamp.SetKey(object.AttributeTimestamp) Creator: *issuer,
timestamp.SetValue(created) Container: *idCnr,
Time: created,
filename := object.NewAttribute() Filename: strconv.FormatInt(created.Unix(), 10) + "_access.box",
filename.SetKey(object.AttributeFileName) ExpirationEpoch: expiration,
filename.SetValue(created + "_access.box") Payload: data,
})
expirationAttr := object.NewAttribute()
expirationAttr.SetKey("__NEOFS__EXPIRATION_EPOCH")
expirationAttr.SetValue(strconv.FormatUint(expiration, 10))
raw := object.NewRaw()
raw.SetContainerID(cid)
raw.SetOwnerID(issuer)
raw.SetAttributes(filename, timestamp, expirationAttr)
raw.SetPayload(data)
oid, err := c.pool.PutObject(ctx, *raw.Object(), nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
addr := address.NewAddress() addr := address.NewAddress()
addr.SetObjectID(oid) addr.SetObjectID(idObj)
addr.SetContainerID(cid) addr.SetContainerID(idCnr)
return addr, nil return addr, nil
} }

589
internal/neofs/neofs.go Normal file
View file

@ -0,0 +1,589 @@
package neofs
import (
"bytes"
"context"
"crypto/ecdsa"
"encoding/binary"
"errors"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/authmate"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/acl"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/token"
)
// NeoFS represents virtual connection to the NeoFS network.
// It is used to provide an interface to dependent packages
// which work with NeoFS.
type NeoFS struct {
pool pool.Pool
}
// SetConnectionPool binds underlying pool.Pool. Must be
// called on initialization stage before any usage.
func (x *NeoFS) SetConnectionPool(p pool.Pool) {
x.pool = p
}
// NetworkState implements authmate.NeoFS interface method.
func (x *NeoFS) NetworkState(ctx context.Context) (*authmate.NetworkState, error) {
conn, _, err := x.pool.Connection()
if err != nil {
return nil, fmt.Errorf("get connection from pool: %w", err)
}
res, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{})
if err != nil {
return nil, fmt.Errorf("get network info via client: %w", err)
}
networkInfo := res.Info()
var durEpoch uint64
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "EpochDuration" {
data := make([]byte, 8)
copy(data, parameter.Value())
durEpoch = binary.LittleEndian.Uint64(data)
return true
}
return false
})
if durEpoch == 0 {
return nil, errors.New("epoch duration is missing or zero")
}
return &authmate.NetworkState{
Epoch: networkInfo.CurrentEpoch(),
BlockDuration: networkInfo.MsPerBlock(),
EpochDuration: durEpoch,
}, nil
}
// Container reads container by ID using connection pool. Returns exact one non-nil value.
func (x *NeoFS) Container(ctx context.Context, idCnr cid.ID) (*container.Container, error) {
res, err := x.pool.GetContainer(ctx, &idCnr)
if err != nil {
return nil, fmt.Errorf("read container via connection pool: %w", err)
}
return res, nil
}
// ContainerExists implements authmate.NeoFS interface method.
func (x *NeoFS) ContainerExists(ctx context.Context, idCnr cid.ID) error {
_, err := x.pool.GetContainer(ctx, &idCnr)
if err != nil {
return fmt.Errorf("get container via connection pool: %w", err)
}
return nil
}
// PrmContainerCreate groups parameters of CreateContainer operation.
type PrmContainerCreate struct {
// NeoFS identifier of the container creator.
Creator owner.ID
// Container placement policy.
Policy netmap.PlacementPolicy
// Name for the container.
Name string
// Time when container is created.
Time time.Time
// Basic ACL of the container.
BasicACL acl.BasicACL
// Token of the container's creation session (optional, nil means session absence).
SessionToken *session.Token
// Attribute for LocationConstraint parameter (optional).
LocationConstraintAttribute *container.Attribute
}
// CreateContainer constructs new container from the parameters and saves it in NeoFS
// using connection pool. Returns any error encountered which prevent the container
// to be saved.
func (x *NeoFS) CreateContainer(ctx context.Context, prm PrmContainerCreate) (*cid.ID, error) {
// fill container structure
cnrOptions := []container.Option{
container.WithPolicy(&prm.Policy),
container.WithOwnerID(&prm.Creator),
container.WithCustomBasicACL(prm.BasicACL),
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)),
}
if prm.Name != "" {
cnrOptions = append(cnrOptions, container.WithAttribute(container.AttributeName, prm.Name))
}
if prm.LocationConstraintAttribute != nil {
cnrOptions = append(cnrOptions, container.WithAttribute(
prm.LocationConstraintAttribute.Key(),
prm.LocationConstraintAttribute.Value(),
))
}
cnr := container.New(cnrOptions...)
cnr.SetSessionToken(prm.SessionToken)
if prm.Name != "" {
container.SetNativeName(cnr, prm.Name)
}
// send request to save the container
idCnr, err := x.pool.PutContainer(ctx, cnr)
if err != nil {
return nil, fmt.Errorf("save container via connection pool: %w", err)
}
// wait the container to be persisted
err = x.pool.WaitForContainerPresence(ctx, idCnr, pool.DefaultPollingParams())
if err != nil {
return nil, fmt.Errorf("wait for container to be saved: %w", err)
}
return idCnr, nil
}
// UserContainers reads list of user containers from NeoFS using connection pool.
// Returns any error encountered which prevent the containers to be listed.
func (x *NeoFS) UserContainers(ctx context.Context, id owner.ID) ([]cid.ID, error) {
r, err := x.pool.ListContainers(ctx, &id)
if err != nil {
return nil, fmt.Errorf("list user containers via connection pool: %w", err)
}
res := make([]cid.ID, len(r))
for i := range r {
res[i] = *r[i]
}
return res, nil
}
// SetContainerEACL saves eACL table of the container in NeoFS using connection pool.
// Returns any error encountered which prevented the eACL to be saved.
func (x *NeoFS) SetContainerEACL(ctx context.Context, table eacl.Table) error {
err := x.pool.SetEACL(ctx, &table)
if err != nil {
return fmt.Errorf("save eACL via connection pool: %w", err)
}
return err
}
// ContainerEACL reads eACL table of the container from NeoFS using connection pool.
// Returns any error encountered which prevented the eACL to be read.
func (x *NeoFS) ContainerEACL(ctx context.Context, id cid.ID) (*eacl.Table, error) {
res, err := x.pool.GetEACL(ctx, &id)
if err != nil {
return nil, fmt.Errorf("read eACL via connection pool: %w", err)
}
return res, nil
}
// DeleteContainer marks container to be removed from NeoFS using connection pool.
// Returns any error encountered which prevented removal request to be sent.
func (x *NeoFS) DeleteContainer(ctx context.Context, id cid.ID, token *session.Token) error {
err := x.pool.DeleteContainer(ctx, &id, pool.WithSession(token))
if err != nil {
return fmt.Errorf("delete container via connection pool: %w", err)
}
return nil
}
type PrmObjectCreate struct {
// NeoFS identifier of the object creator.
Creator owner.ID
// NeoFS container to store the object.
Container cid.ID
// Object creation time.
Time time.Time
// Associated filename (optional).
Filename string
// Last NeoFS epoch of the object lifetime (optional).
ExpirationEpoch uint64
// Full payload size (optional).
PayloadSize uint64
// Key-value object attributes.
Attributes [][2]string
// Object payload encapsulated in io.Reader primitive.
Payload io.Reader
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
BearerToken *token.BearerToken
// Private key used for the operation if BearerToken is missing (in this case non-nil).
PrivateKey *ecdsa.PrivateKey
}
// CreateObject creates and saves a parameterized object in the specified
// NeoFS container from a specific user. Returns ID of the saved object.
//
// Returns exactly one non-nil value. Returns any error encountered which
// prevented the object to be created.
func (x *NeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (*oid.ID, error) {
attrNum := len(prm.Attributes) + 1 // + creation time
if prm.Filename != "" {
attrNum++
}
if prm.ExpirationEpoch > 0 {
attrNum++
}
attrs := make([]*object.Attribute, 0, attrNum)
var a *object.Attribute
a = object.NewAttribute()
a.SetKey(object.AttributeTimestamp)
a.SetValue(strconv.FormatInt(prm.Time.Unix(), 10))
attrs = append(attrs, a)
for i := range prm.Attributes {
a = object.NewAttribute()
a.SetKey(prm.Attributes[i][0])
a.SetValue(prm.Attributes[i][1])
attrs = append(attrs, a)
}
if prm.Filename != "" {
a = object.NewAttribute()
a.SetKey(object.AttributeFileName)
a.SetValue(prm.Filename)
attrs = append(attrs, a)
}
if prm.ExpirationEpoch > 0 {
a = object.NewAttribute()
a.SetKey("__NEOFS__EXPIRATION_EPOCH")
a.SetValue(strconv.FormatUint(prm.ExpirationEpoch, 10))
attrs = append(attrs, a)
}
raw := object.NewRaw()
raw.SetContainerID(&prm.Container)
raw.SetOwnerID(&prm.Creator)
raw.SetAttributes(attrs...)
raw.SetPayloadSize(prm.PayloadSize)
var callOpt pool.CallOption
if prm.BearerToken != nil {
callOpt = pool.WithBearer(prm.BearerToken)
} else {
callOpt = pool.WithKey(prm.PrivateKey)
}
idObj, err := x.pool.PutObject(ctx, *raw.Object(), prm.Payload, callOpt)
if err != nil {
return nil, fmt.Errorf("save object via connection pool: %w", err)
}
return idObj, nil
}
// SelectObjects selects user objects which match specified filters from the NeoFS container
// using connection pool.
//
// Returns any error encountered which prevented the selection to be finished.
// Returns layer.ErrAccessDenied on access violation.
func (x *NeoFS) SelectObjects(ctx context.Context, prm layer.PrmObjectSelect) ([]oid.ID, error) {
var filters object.SearchFilters
filters.AddRootFilter()
if prm.ExactAttribute[0] != "" {
filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual)
}
if prm.FilePrefix != "" {
filters.AddFilter(object.AttributeFileName, prm.FilePrefix, object.MatchCommonPrefix)
}
var callOpt pool.CallOption
if prm.BearerToken != nil {
callOpt = pool.WithBearer(prm.BearerToken)
} else {
callOpt = pool.WithKey(prm.PrivateKey)
}
res, err := x.pool.SearchObjects(ctx, prm.Container, filters, callOpt)
if err != nil {
return nil, fmt.Errorf("init object search via connection pool: %w", err)
}
defer res.Close()
var num, read int
buf := make([]oid.ID, 10)
for {
num, err = res.Read(buf[read:])
if num > 0 {
read += num
buf = append(buf, oid.ID{})
buf = buf[:cap(buf)]
}
if err != nil {
if errors.Is(err, io.EOF) {
break
}
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return nil, layer.ErrAccessDenied
}
return nil, fmt.Errorf("read object list: %w", err)
}
}
return buf[:read], nil
}
// wraps io.ReadCloser and transforms Read errors related to access violation
// to layer.ErrAccessDenied.
type payloadReader struct {
io.ReadCloser
}
func (x payloadReader) Read(p []byte) (int, error) {
n, err := x.ReadCloser.Read(p)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return n, layer.ErrAccessDenied
}
}
return n, err
}
// ReadObject reads object part from the NeoFS container by identifier using connection pool:
// * if with header only, then HeadObject is called;
// * if with non-zero payload range only, then ObjectRange is called;
// * else GetObject is called.
//
// Returns any error encountered which prevented the object to be read.
// Returns layer.ErrAccessDenied on access violation.
func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer.ObjectPart, error) {
var addr address.Address
addr.SetContainerID(&prm.Container)
addr.SetObjectID(&prm.Object)
var callOpt pool.CallOption
if prm.BearerToken != nil {
callOpt = pool.WithBearer(prm.BearerToken)
} else {
callOpt = pool.WithKey(prm.PrivateKey)
}
if prm.WithHeader {
if prm.WithPayload {
res, err := x.pool.GetObject(ctx, addr, callOpt)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return nil, layer.ErrAccessDenied
}
return nil, fmt.Errorf("init full object reading via connection pool: %w", err)
}
return &layer.ObjectPart{
Head: &res.Header,
Payload: res.Payload,
}, nil
}
hdr, err := x.pool.HeadObject(ctx, addr, callOpt)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return nil, layer.ErrAccessDenied
}
return nil, fmt.Errorf("read object header via connection pool: %w", err)
}
return &layer.ObjectPart{
Head: hdr,
}, nil
} else if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 {
res, err := x.pool.GetObject(ctx, addr, callOpt)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return nil, layer.ErrAccessDenied
}
return nil, fmt.Errorf("init full payload range reading via connection pool: %w", err)
}
return &layer.ObjectPart{
Payload: res.Payload,
}, nil
}
res, err := x.pool.ObjectRange(ctx, addr, prm.PayloadRange[0], prm.PayloadRange[1], callOpt)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return nil, layer.ErrAccessDenied
}
return nil, fmt.Errorf("init payload range reading via connection pool: %w", err)
}
return &layer.ObjectPart{
Payload: payloadReader{res},
}, nil
}
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
// Successful return does not guarantee the actual removal.
//
// Returns ErrAccessDenied on remove access violation.
//
// Returns any error encountered which prevented the removal request to be sent.
func (x *NeoFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) error {
var addr address.Address
addr.SetContainerID(&prm.Container)
addr.SetObjectID(&prm.Object)
var callOpt pool.CallOption
if prm.BearerToken != nil {
callOpt = pool.WithBearer(prm.BearerToken)
} else {
callOpt = pool.WithKey(prm.PrivateKey)
}
err := x.pool.DeleteObject(ctx, addr, callOpt)
if err != nil {
// TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
return layer.ErrAccessDenied
}
return fmt.Errorf("mark object removal via connection pool: %w", err)
}
return nil
}
// AuthmateNeoFS is a mediator which implements authmate.NeoFS through NeoFS.
type AuthmateNeoFS struct {
NeoFS
}
func (x *AuthmateNeoFS) CreateContainer(ctx context.Context, prm authmate.PrmContainerCreate) (*cid.ID, error) {
return x.NeoFS.CreateContainer(ctx, PrmContainerCreate{
Creator: prm.Owner,
Policy: prm.Policy,
Name: prm.FriendlyName,
Time: time.Now(),
BasicACL: 0b0011_1100_1000_1100_1000_1100_1100_1110, // 0x3C8C8CCE
})
}
func (x *AuthmateNeoFS) ReadObjectPayload(ctx context.Context, addr address.Address) ([]byte, error) {
res, err := x.NeoFS.ReadObject(ctx, layer.PrmObjectRead{
Container: *addr.ContainerID(),
Object: *addr.ObjectID(),
WithPayload: true,
})
if err != nil {
return nil, err
}
defer res.Payload.Close()
return io.ReadAll(res.Payload)
}
func (x *AuthmateNeoFS) CreateObject(ctx context.Context, prm tokens.PrmObjectCreate) (*oid.ID, error) {
return x.NeoFS.CreateObject(ctx, PrmObjectCreate{
Creator: prm.Creator,
Container: prm.Container,
Time: prm.Time,
Filename: prm.Filename,
ExpirationEpoch: prm.ExpirationEpoch,
Payload: bytes.NewReader(prm.Payload),
})
}
// SystemDNS reads "SystemDNS" network parameter of the NeoFS.
//
// Returns exactly on non-zero value. Returns any error encountered
// which prevented the parameter to be read.
func (x *NeoFS) SystemDNS(ctx context.Context) (string, error) {
conn, _, err := x.pool.Connection()
if err != nil {
return "", fmt.Errorf("get connection from the pool: %w", err)
}
var prmCli client.PrmNetworkInfo
res, err := conn.NetworkInfo(ctx, prmCli)
if err != nil {
return "", fmt.Errorf("read network info via client: %w", err)
}
var domain string
res.Info().NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "SystemDNS" {
domain = string(parameter.Value())
return true
}
return false
})
if domain == "" {
return "", errors.New("system DNS parameter not found or empty")
}
return domain, nil
}