forked from TrueCloudLab/frostfs-s3-gw
[#346] *: Refactor communication with NeoFS at the protocol level
Make `tokens`, `authmate` and `layer` packages to depend from locally defined `NeoFS` interface of the virtual connection to NeoFS network. Create internal `neofs` package and implement these interfaces through `pool.Pool` there. Implement mediators between `NeoFS` interfaces and `neofs.NeoFS` implementation. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich
LeL
parent
34a221c5c9
commit
cd64f41ce8
14 changed files with 1348 additions and 606 deletions
|
|
@ -22,8 +22,6 @@ import (
|
|||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||
|
|
@ -44,12 +42,6 @@ type (
|
|||
cli tokens.Credentials
|
||||
}
|
||||
|
||||
// Params stores node connection parameters.
|
||||
Params struct {
|
||||
Pool pool.Pool
|
||||
Logger *zap.Logger
|
||||
}
|
||||
|
||||
prs int
|
||||
|
||||
authHeader struct {
|
||||
|
|
@ -82,9 +74,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
|
|||
var _ io.ReadSeeker = prs(0)
|
||||
|
||||
// New creates an instance of AuthCenter.
|
||||
func New(conns pool.Pool, key *keys.PrivateKey, config *cache.Config) Center {
|
||||
func New(neoFS tokens.NeoFS, key *keys.PrivateKey, config *cache.Config) Center {
|
||||
return ¢er{
|
||||
cli: tokens.New(conns, key, config),
|
||||
cli: tokens.New(neoFS, key, config),
|
||||
reg: ®expSubmatcher{re: authorizationFieldRegexp},
|
||||
postReg: ®expSubmatcher{re: postPolicyCredentialRegexp},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ import (
|
|||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
@ -30,21 +29,21 @@ type (
|
|||
|
||||
const locationConstraintAttr = ".s3-location-constraint"
|
||||
|
||||
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInfo, error) {
|
||||
func (n *layer) containerInfo(ctx context.Context, idCnr *cid.ID) (*data.BucketInfo, error) {
|
||||
var (
|
||||
err error
|
||||
res *container.Container
|
||||
rid = api.GetRequestID(ctx)
|
||||
|
||||
info = &data.BucketInfo{
|
||||
CID: cid,
|
||||
Name: cid.String(),
|
||||
CID: idCnr,
|
||||
Name: idCnr.String(),
|
||||
}
|
||||
)
|
||||
res, err = n.pool.GetContainer(ctx, cid, n.CallOptions(ctx)...)
|
||||
res, err = n.neoFS.Container(ctx, *idCnr)
|
||||
if err != nil {
|
||||
n.log.Error("could not fetch container",
|
||||
zap.Stringer("cid", cid),
|
||||
zap.Stringer("cid", idCnr),
|
||||
zap.String("request_id", rid),
|
||||
zap.Error(err))
|
||||
|
||||
|
|
@ -65,7 +64,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInf
|
|||
unix, err := strconv.ParseInt(attr.Value(), 10, 64)
|
||||
if err != nil {
|
||||
n.log.Error("could not parse container creation time",
|
||||
zap.Stringer("cid", cid),
|
||||
zap.Stringer("cid", idCnr),
|
||||
zap.String("request_id", rid),
|
||||
zap.String("created_at", val),
|
||||
zap.Error(err))
|
||||
|
|
@ -81,7 +80,7 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*data.BucketInf
|
|||
|
||||
if err := n.bucketCache.Put(info); err != nil {
|
||||
n.log.Warn("could not put bucket info into cache",
|
||||
zap.Stringer("cid", cid),
|
||||
zap.Stringer("cid", idCnr),
|
||||
zap.String("bucket_name", info.Name),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
|
@ -93,20 +92,20 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
var (
|
||||
err error
|
||||
own = n.Owner(ctx)
|
||||
res []*cid.ID
|
||||
res []cid.ID
|
||||
rid = api.GetRequestID(ctx)
|
||||
)
|
||||
res, err = n.pool.ListContainers(ctx, own, n.CallOptions(ctx)...)
|
||||
res, err = n.neoFS.UserContainers(ctx, *own)
|
||||
if err != nil {
|
||||
n.log.Error("could not fetch container",
|
||||
n.log.Error("could not list user containers",
|
||||
zap.String("request_id", rid),
|
||||
zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
list := make([]*data.BucketInfo, 0, len(res))
|
||||
for _, cid := range res {
|
||||
info, err := n.containerInfo(ctx, cid)
|
||||
for i := range res {
|
||||
info, err := n.containerInfo(ctx, &res[i])
|
||||
if err != nil {
|
||||
n.log.Error("could not fetch container info",
|
||||
zap.String("request_id", rid),
|
||||
|
|
@ -130,28 +129,23 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*ci
|
|||
LocationConstraint: p.LocationConstraint,
|
||||
}
|
||||
|
||||
options := []container.Option{
|
||||
container.WithPolicy(p.Policy),
|
||||
container.WithCustomBasicACL(acl.BasicACL(p.ACL)),
|
||||
container.WithAttribute(container.AttributeName, p.Name),
|
||||
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(bktInfo.Created.Unix(), 10)),
|
||||
}
|
||||
var locConstAttr *container.Attribute
|
||||
|
||||
if p.LocationConstraint != "" {
|
||||
options = append(options, container.WithAttribute(locationConstraintAttr, p.LocationConstraint))
|
||||
locConstAttr = container.NewAttribute()
|
||||
locConstAttr.SetKey(locationConstraintAttr)
|
||||
locConstAttr.SetValue(p.LocationConstraint)
|
||||
}
|
||||
|
||||
cnr := container.New(options...)
|
||||
container.SetNativeName(cnr, p.Name)
|
||||
|
||||
cnr.SetSessionToken(p.SessionToken)
|
||||
cnr.SetOwnerID(bktInfo.Owner)
|
||||
|
||||
if bktInfo.CID, err = n.pool.PutContainer(ctx, cnr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = n.pool.WaitForContainerPresence(ctx, bktInfo.CID, pool.DefaultPollingParams()); err != nil {
|
||||
if bktInfo.CID, err = n.neoFS.CreateContainer(ctx, PrmContainerCreate{
|
||||
Creator: *bktInfo.Owner,
|
||||
Policy: *p.Policy,
|
||||
Name: p.Name,
|
||||
SessionToken: p.SessionToken,
|
||||
Time: bktInfo.Created,
|
||||
BasicACL: acl.BasicACL(p.ACL),
|
||||
LocationConstraintAttribute: locConstAttr,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -172,21 +166,20 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*ci
|
|||
func (n *layer) setContainerEACLTable(ctx context.Context, cid *cid.ID, table *eacl.Table) error {
|
||||
table.SetCID(cid)
|
||||
|
||||
var sessionToken *session.Token
|
||||
boxData, err := GetBoxData(ctx)
|
||||
if err == nil {
|
||||
sessionToken = boxData.Gate.SessionTokenForSetEACL()
|
||||
table.SetSessionToken(boxData.Gate.SessionTokenForSetEACL())
|
||||
}
|
||||
|
||||
if err := n.pool.SetEACL(ctx, table, pool.WithSession(sessionToken)); err != nil {
|
||||
if err := n.neoFS.SetContainerEACL(ctx, *table); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return n.waitEACLPresence(ctx, cid, table, defaultWaitParams())
|
||||
return n.waitEACLPresence(ctx, *cid, table, defaultWaitParams())
|
||||
}
|
||||
|
||||
func (n *layer) GetContainerEACL(ctx context.Context, cid *cid.ID) (*eacl.Table, error) {
|
||||
return n.pool.GetEACL(ctx, cid)
|
||||
return n.neoFS.ContainerEACL(ctx, *cid)
|
||||
}
|
||||
|
||||
type waitParams struct {
|
||||
|
|
@ -201,7 +194,7 @@ func defaultWaitParams() *waitParams {
|
|||
}
|
||||
}
|
||||
|
||||
func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.Table, params *waitParams) error {
|
||||
func (n *layer) waitEACLPresence(ctx context.Context, cid cid.ID, table *eacl.Table, params *waitParams) error {
|
||||
exp, err := table.Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't marshal eacl: %w", err)
|
||||
|
|
@ -213,6 +206,8 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
|
|||
defer ticker.Stop()
|
||||
wdone := wctx.Done()
|
||||
done := ctx.Done()
|
||||
var eaclTable *eacl.Table
|
||||
var got []byte
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
|
|
@ -220,10 +215,13 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
|
|||
case <-wdone:
|
||||
return wctx.Err()
|
||||
case <-ticker.C:
|
||||
eaclTable, err := n.pool.GetEACL(ctx, cid)
|
||||
eaclTable, err = n.neoFS.ContainerEACL(ctx, cid)
|
||||
if err == nil {
|
||||
got, err := eaclTable.Marshal()
|
||||
if err == nil && bytes.Equal(exp, got) {
|
||||
got, err = eaclTable.Marshal()
|
||||
if err != nil {
|
||||
// not expected, but if occurred - doesn't make sense to continue
|
||||
return fmt.Errorf("marshal received eACL: %w", err)
|
||||
} else if bytes.Equal(exp, got) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -232,11 +230,11 @@ func (n *layer) waitEACLPresence(ctx context.Context, cid *cid.ID, table *eacl.T
|
|||
}
|
||||
}
|
||||
|
||||
func (n *layer) deleteContainer(ctx context.Context, cid *cid.ID) error {
|
||||
func (n *layer) deleteContainer(ctx context.Context, idCnr *cid.ID) error {
|
||||
var sessionToken *session.Token
|
||||
boxData, err := GetBoxData(ctx)
|
||||
if err == nil {
|
||||
sessionToken = boxData.Gate.SessionTokenForDelete()
|
||||
}
|
||||
return n.pool.DeleteContainer(ctx, cid, pool.WithSession(sessionToken))
|
||||
return n.neoFS.DeleteContainer(ctx, *idCnr, sessionToken)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
|
|
@ -18,6 +19,8 @@ import (
|
|||
"github.com/nspcc-dev/neofs-s3-gw/api/notifications"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/acl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
|
|
@ -27,12 +30,216 @@ import (
|
|||
"github.com/nspcc-dev/neofs-sdk-go/owner"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/token"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
|
||||
type PrmContainerCreate struct {
|
||||
// NeoFS identifier of the container creator.
|
||||
Creator owner.ID
|
||||
|
||||
// Container placement policy.
|
||||
Policy netmap.PlacementPolicy
|
||||
|
||||
// Name for the container.
|
||||
Name string
|
||||
|
||||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Token
|
||||
|
||||
// Time when container is created.
|
||||
Time time.Time
|
||||
|
||||
// Basic ACL of the container.
|
||||
BasicACL acl.BasicACL
|
||||
|
||||
// Attribute for LocationConstraint parameter (optional).
|
||||
LocationConstraintAttribute *container.Attribute
|
||||
}
|
||||
|
||||
// PrmAuth groups authentication parameters for the NeoFS operation.
|
||||
type PrmAuth struct {
|
||||
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
||||
BearerToken *token.BearerToken
|
||||
|
||||
// Private key used for the operation if BearerToken is missing (in this case non-nil).
|
||||
PrivateKey *ecdsa.PrivateKey
|
||||
}
|
||||
|
||||
// PrmObjectSelect groups parameters of NeoFS.SelectObjects operation.
|
||||
type PrmObjectSelect struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
||||
// Container to select the objects from.
|
||||
Container cid.ID
|
||||
|
||||
// Key-value object attribute which should exactly be
|
||||
// presented in selected objects. Optional, empty key means any.
|
||||
ExactAttribute [2]string
|
||||
|
||||
// File prefix of the selected objects. Optional, empty value means any.
|
||||
FilePrefix string
|
||||
}
|
||||
|
||||
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
|
||||
type PrmObjectRead struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
||||
// Container to read the object header from.
|
||||
Container cid.ID
|
||||
|
||||
// ID of the object for which to read the header.
|
||||
Object oid.ID
|
||||
|
||||
// Flag to read object header.
|
||||
WithHeader bool
|
||||
|
||||
// Flag to read object payload. False overlaps payload range.
|
||||
WithPayload bool
|
||||
|
||||
// Offset-length range of the object payload to be read.
|
||||
PayloadRange [2]uint64
|
||||
}
|
||||
|
||||
// ObjectPart represents partially read NeoFS object.
|
||||
type ObjectPart struct {
|
||||
// Object header with optional in-memory payload part.
|
||||
Head *object.Object
|
||||
|
||||
// Object payload part encapsulated in io.Reader primitive.
|
||||
// Returns ErrAccessDenied on read access violation.
|
||||
Payload io.ReadCloser
|
||||
}
|
||||
|
||||
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
|
||||
type PrmObjectCreate struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
||||
// Container to store the object.
|
||||
Container cid.ID
|
||||
|
||||
// NeoFS identifier of the object creator.
|
||||
Creator owner.ID
|
||||
|
||||
// Key-value object attributes.
|
||||
Attributes [][2]string
|
||||
|
||||
// Full payload size (optional).
|
||||
PayloadSize uint64
|
||||
|
||||
// Associated filename (optional).
|
||||
Filename string
|
||||
|
||||
// Object payload encapsulated in io.Reader primitive.
|
||||
Payload io.Reader
|
||||
}
|
||||
|
||||
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
|
||||
type PrmObjectDelete struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
||||
// Container to delete the object from.
|
||||
Container cid.ID
|
||||
|
||||
// Identifier of the removed object.
|
||||
Object oid.ID
|
||||
}
|
||||
|
||||
// ErrAccessDenied is returned from NeoFS in case of access violation.
|
||||
var ErrAccessDenied = stderrors.New("access denied")
|
||||
|
||||
// NeoFS represents virtual connection to NeoFS network.
|
||||
type NeoFS interface {
|
||||
// CreateContainer creates and saves parameterized container in NeoFS.
|
||||
// Returns ID of the saved container.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the container to be created.
|
||||
CreateContainer(context.Context, PrmContainerCreate) (*cid.ID, error)
|
||||
|
||||
// Container reads container from NeoFS by ID.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the container to be read.
|
||||
Container(context.Context, cid.ID) (*container.Container, error)
|
||||
|
||||
// UserContainers reads list of the containers owned by specified user.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the containers to be listed.
|
||||
UserContainers(context.Context, owner.ID) ([]cid.ID, error)
|
||||
|
||||
// SetContainerEACL saves eACL table of the container in NeoFS.
|
||||
//
|
||||
// Returns any error encountered which prevented the eACL to be saved.
|
||||
SetContainerEACL(context.Context, eacl.Table) error
|
||||
|
||||
// ContainerEACL reads container eACL from NeoFS by container ID.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the eACL to be read.
|
||||
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
|
||||
|
||||
// DeleteContainer marks the container to be removed from NeoFS by ID.
|
||||
// Request is sent within session if the session token is specified.
|
||||
// Successful return does not guarantee the actual removal.
|
||||
//
|
||||
// Returns any error encountered which prevented the removal request to be sent.
|
||||
DeleteContainer(context.Context, cid.ID, *session.Token) error
|
||||
|
||||
// SelectObjects perform object selection from the NeoFS container according
|
||||
// to specified parameters. Selects user objects only.
|
||||
//
|
||||
// Returns ErrAccessDenied on selection access violation.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the objects to be selected.
|
||||
SelectObjects(context.Context, PrmObjectSelect) ([]oid.ID, error)
|
||||
|
||||
// ReadObject reads part of the object from the NeoFS container by identifier.
|
||||
// Exact part is returned according to the parameters:
|
||||
// * with header only: empty payload (both in-mem and reader parts are nil);
|
||||
// * with payload only: header is nil (zero range means full payload);
|
||||
// * with header and payload: full in-mem object, payload reader is nil.
|
||||
//
|
||||
// WithHeader or WithPayload is true. Range length is positive if offset is positive.
|
||||
//
|
||||
// Payload reader should be closed if it is no longer needed.
|
||||
//
|
||||
// Returns ErrAccessDenied on read access violation.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the object header to be read.
|
||||
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
|
||||
|
||||
// CreateObject creates and saves parameterized object in the NeoFS container.
|
||||
// Returns ID of the saved object.
|
||||
//
|
||||
// Creation time should be written into object (UTC).
|
||||
//
|
||||
// Returns ErrAccessDenied on write access violation.
|
||||
//
|
||||
// Returns exactly one non-nil value. Returns any error encountered which
|
||||
// prevented the container to be created.
|
||||
CreateObject(context.Context, PrmObjectCreate) (*oid.ID, error)
|
||||
|
||||
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
|
||||
// Successful return does not guarantee the actual removal.
|
||||
//
|
||||
// Returns ErrAccessDenied on remove access violation.
|
||||
//
|
||||
// Returns any error encountered which prevented the removal request to be sent.
|
||||
DeleteObject(context.Context, PrmObjectDelete) error
|
||||
}
|
||||
|
||||
type (
|
||||
layer struct {
|
||||
pool pool.Pool
|
||||
neoFS NeoFS
|
||||
log *zap.Logger
|
||||
anonKey AnonymousKey
|
||||
resolver *resolver.BucketResolver
|
||||
|
|
@ -66,14 +273,6 @@ type (
|
|||
System *cache.Config
|
||||
}
|
||||
|
||||
// Params stores basic API parameters.
|
||||
Params struct {
|
||||
Pool pool.Pool
|
||||
Logger *zap.Logger
|
||||
Timeout time.Duration
|
||||
Key *ecdsa.PrivateKey
|
||||
}
|
||||
|
||||
// GetObjectParams stores object get request parameters.
|
||||
GetObjectParams struct {
|
||||
Range *RangeParams
|
||||
|
|
@ -183,15 +382,8 @@ type (
|
|||
TagSet map[string]string
|
||||
}
|
||||
|
||||
// NeoFS provides basic NeoFS interface.
|
||||
NeoFS interface {
|
||||
Get(ctx context.Context, address *address.Address) (*object.Object, error)
|
||||
}
|
||||
|
||||
// Client provides S3 API client interface.
|
||||
Client interface {
|
||||
NeoFS
|
||||
|
||||
EphemeralKey() *keys.PublicKey
|
||||
|
||||
PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*data.ObjectInfo, error)
|
||||
|
|
@ -262,9 +454,9 @@ func DefaultCachesConfigs() *CachesConfig {
|
|||
|
||||
// NewLayer creates instance of layer. It checks credentials
|
||||
// and establishes gRPC connection with node.
|
||||
func NewLayer(log *zap.Logger, conns pool.Pool, config *Config) Client {
|
||||
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
||||
return &layer{
|
||||
pool: conns,
|
||||
neoFS: neoFS,
|
||||
log: log,
|
||||
anonKey: config.AnonKey,
|
||||
resolver: config.Resolver,
|
||||
|
|
@ -293,17 +485,26 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
|
|||
|
||||
// Owner returns owner id from BearerToken (context) or from client owner.
|
||||
func (n *layer) Owner(ctx context.Context) *owner.ID {
|
||||
if data, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && data != nil && data.Gate != nil {
|
||||
return data.Gate.BearerToken.Issuer()
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
|
||||
return bd.Gate.BearerToken.Issuer()
|
||||
}
|
||||
|
||||
return owner.NewIDFromPublicKey((*ecdsa.PublicKey)(n.EphemeralKey()))
|
||||
}
|
||||
|
||||
func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth) {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
|
||||
prm.BearerToken = bd.Gate.BearerToken
|
||||
return
|
||||
}
|
||||
|
||||
prm.PrivateKey = &n.anonKey.Key.PrivateKey
|
||||
}
|
||||
|
||||
// CallOptions returns []pool.CallOption options: client.WithBearer or client.WithKey (if request is anonymous).
|
||||
func (n *layer) CallOptions(ctx context.Context) []pool.CallOption {
|
||||
if data, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && data != nil && data.Gate != nil {
|
||||
return []pool.CallOption{pool.WithBearer(data.Gate.BearerToken)}
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
|
||||
return []pool.CallOption{pool.WithBearer(bd.Gate.BearerToken)}
|
||||
}
|
||||
|
||||
return []pool.CallOption{pool.WithKey(&n.anonKey.Key.PrivateKey)}
|
||||
|
|
@ -341,14 +542,14 @@ func (n *layer) GetBucketACL(ctx context.Context, name string) (*BucketACL, erro
|
|||
return nil, err
|
||||
}
|
||||
|
||||
eacl, err := n.GetContainerEACL(ctx, inf.CID)
|
||||
eACL, err := n.GetContainerEACL(ctx, inf.CID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BucketACL{
|
||||
Info: inf,
|
||||
EACL: eacl,
|
||||
EACL: eACL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -308,8 +308,8 @@ func (n *layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUpload
|
|||
}
|
||||
|
||||
f := &findParams{
|
||||
filters: []filter{{attr: UploadPartNumberAttributeName, val: "0"}},
|
||||
cid: p.Bkt.CID,
|
||||
attr: [2]string{UploadPartNumberAttributeName, "0"},
|
||||
cid: p.Bkt.CID,
|
||||
}
|
||||
|
||||
ids, err := n.objectSearch(ctx, f)
|
||||
|
|
|
|||
|
|
@ -3,10 +3,8 @@ package layer
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -18,20 +16,14 @@ import (
|
|||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/owner"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
findParams struct {
|
||||
filters []filter
|
||||
cid *cid.ID
|
||||
prefix string
|
||||
}
|
||||
|
||||
filter struct {
|
||||
attr string
|
||||
val string
|
||||
attr [2]string
|
||||
cid *cid.ID
|
||||
prefix string
|
||||
}
|
||||
|
||||
getParams struct {
|
||||
|
|
@ -76,54 +68,25 @@ type (
|
|||
|
||||
func (n *layer) objectSearchByName(ctx context.Context, cid *cid.ID, filename string) ([]oid.ID, error) {
|
||||
f := &findParams{
|
||||
filters: []filter{{attr: object.AttributeFileName, val: filename}},
|
||||
cid: cid,
|
||||
prefix: "",
|
||||
attr: [2]string{object.AttributeFileName, filename},
|
||||
cid: cid,
|
||||
}
|
||||
return n.objectSearch(ctx, f)
|
||||
}
|
||||
|
||||
// objectSearch returns all available objects by search params.
|
||||
func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]oid.ID, error) {
|
||||
var filters object.SearchFilters
|
||||
filters.AddRootFilter()
|
||||
|
||||
for _, filter := range p.filters {
|
||||
filters.AddFilter(filter.attr, filter.val, object.MatchStringEqual)
|
||||
prm := PrmObjectSelect{
|
||||
Container: *p.cid,
|
||||
ExactAttribute: p.attr,
|
||||
FilePrefix: p.prefix,
|
||||
}
|
||||
|
||||
if p.prefix != "" {
|
||||
filters.AddFilter(object.AttributeFileName, p.prefix, object.MatchCommonPrefix)
|
||||
}
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
res, err := n.pool.SearchObjects(ctx, *p.cid, filters, n.CallOptions(ctx)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init searching using client: %w", err)
|
||||
}
|
||||
res, err := n.neoFS.SelectObjects(ctx, prm)
|
||||
|
||||
defer res.Close()
|
||||
|
||||
var num, read int
|
||||
buf := make([]oid.ID, 10)
|
||||
|
||||
for {
|
||||
num, err = res.Read(buf[read:])
|
||||
if num > 0 {
|
||||
read += num
|
||||
buf = append(buf, oid.ID{})
|
||||
buf = buf[:cap(buf)]
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
return nil, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
return buf[:read], nil
|
||||
return res, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
||||
func newAddress(cid *cid.ID, oid *oid.ID) *address.Address {
|
||||
|
|
@ -135,83 +98,69 @@ func newAddress(cid *cid.ID, oid *oid.ID) *address.Address {
|
|||
|
||||
// objectHead returns all object's headers.
|
||||
func (n *layer) objectHead(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) (*object.Object, error) {
|
||||
var addr address.Address
|
||||
prm := PrmObjectRead{
|
||||
Container: *idCnr,
|
||||
Object: *idObj,
|
||||
WithHeader: true,
|
||||
}
|
||||
|
||||
addr.SetContainerID(idCnr)
|
||||
addr.SetObjectID(idObj)
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
obj, err := n.pool.HeadObject(ctx, addr, n.CallOptions(ctx)...)
|
||||
return obj, n.transformNeofsError(ctx, err)
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
||||
return res.Head, nil
|
||||
}
|
||||
|
||||
// writes payload part of the NeoFS object to the provided io.Writer.
|
||||
// Zero range corresponds to full payload (panics if only offset is set).
|
||||
func (n *layer) objectWritePayload(ctx context.Context, p getParams) error {
|
||||
// form object address
|
||||
var a address.Address
|
||||
|
||||
a.SetContainerID(p.cid)
|
||||
a.SetObjectID(p.oid)
|
||||
|
||||
fmt.Println("objectWritePayload", p.cid, p.oid)
|
||||
|
||||
// init payload reader
|
||||
var r io.ReadCloser
|
||||
|
||||
if p.ln+p.off == 0 {
|
||||
res, err := n.pool.GetObject(ctx, a, n.CallOptions(ctx)...)
|
||||
if err != nil {
|
||||
return n.transformNeofsError(ctx, fmt.Errorf("get object using client: %w", err))
|
||||
}
|
||||
|
||||
p.ln = res.Header.PayloadSize()
|
||||
r = res.Payload
|
||||
} else {
|
||||
res, err := n.pool.ObjectRange(ctx, a, p.off, p.ln, n.CallOptions(ctx)...)
|
||||
if err != nil {
|
||||
return n.transformNeofsError(ctx, fmt.Errorf("range object payload using client: %w", err))
|
||||
}
|
||||
|
||||
r = res
|
||||
prm := PrmObjectRead{
|
||||
Container: *p.cid,
|
||||
Object: *p.oid,
|
||||
WithPayload: true,
|
||||
PayloadRange: [2]uint64{p.off, p.ln},
|
||||
}
|
||||
|
||||
defer r.Close()
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
if p.ln > 0 {
|
||||
if p.ln > 4096 { // configure?
|
||||
p.ln = 4096
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
if err == nil {
|
||||
defer res.Payload.Close()
|
||||
|
||||
if p.ln == 0 {
|
||||
p.ln = 4096 // configure?
|
||||
}
|
||||
|
||||
// alloc buffer for copying
|
||||
buf := make([]byte, p.ln) // sync-pool it?
|
||||
|
||||
// copy full payload
|
||||
_, err := io.CopyBuffer(p.w, r, buf)
|
||||
if err != nil {
|
||||
return n.transformNeofsError(ctx, fmt.Errorf("copy payload range: %w", err))
|
||||
}
|
||||
_, err = io.CopyBuffer(p.w, res.Payload, buf)
|
||||
}
|
||||
|
||||
return nil
|
||||
return n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
||||
// objectGet returns an object with payload in the object.
|
||||
func (n *layer) objectGet(ctx context.Context, addr *address.Address) (*object.Object, error) {
|
||||
res, err := n.pool.GetObject(ctx, *addr, n.CallOptions(ctx)...)
|
||||
prm := PrmObjectRead{
|
||||
Container: *addr.ContainerID(),
|
||||
Object: *addr.ObjectID(),
|
||||
WithHeader: true,
|
||||
WithPayload: true,
|
||||
}
|
||||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
||||
defer res.Payload.Close()
|
||||
|
||||
payload, err := io.ReadAll(res.Payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read payload: %w", err)
|
||||
}
|
||||
|
||||
object.NewRawFrom(&res.Header).SetPayload(payload)
|
||||
|
||||
return &res.Header, nil
|
||||
return res.Head, nil
|
||||
}
|
||||
|
||||
// objectPut into NeoFS, took payload from io.Reader.
|
||||
|
|
@ -235,9 +184,24 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec
|
|||
r = d.MultiReader()
|
||||
}
|
||||
}
|
||||
rawObject := formRawObject(p, bkt.CID, own, p.Object)
|
||||
|
||||
id, err := n.pool.PutObject(ctx, *rawObject.Object(), r, n.CallOptions(ctx)...)
|
||||
prm := PrmObjectCreate{
|
||||
Container: *bkt.CID,
|
||||
Creator: *own,
|
||||
PayloadSize: uint64(p.Size),
|
||||
Filename: p.Object,
|
||||
Payload: r,
|
||||
}
|
||||
|
||||
prm.Attributes = make([][2]string, 0, len(p.Header))
|
||||
|
||||
for k, v := range p.Header {
|
||||
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
||||
}
|
||||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
id, err := n.neoFS.CreateObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
|
@ -292,35 +256,6 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec
|
|||
}, nil
|
||||
}
|
||||
|
||||
func formRawObject(p *PutObjectParams, bktID *cid.ID, own *owner.ID, obj string) *object.RawObject {
|
||||
attributes := make([]*object.Attribute, 0, len(p.Header)+2)
|
||||
filename := object.NewAttribute()
|
||||
filename.SetKey(object.AttributeFileName)
|
||||
filename.SetValue(obj)
|
||||
|
||||
createdAt := object.NewAttribute()
|
||||
createdAt.SetKey(object.AttributeTimestamp)
|
||||
createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
|
||||
|
||||
attributes = append(attributes, filename, createdAt)
|
||||
|
||||
for k, v := range p.Header {
|
||||
ua := object.NewAttribute()
|
||||
ua.SetKey(k)
|
||||
ua.SetValue(v)
|
||||
|
||||
attributes = append(attributes, ua)
|
||||
}
|
||||
|
||||
raw := object.NewRaw()
|
||||
raw.SetOwnerID(own)
|
||||
raw.SetContainerID(bktID)
|
||||
raw.SetAttributes(attributes...)
|
||||
raw.SetPayloadSize(uint64(p.Size))
|
||||
|
||||
return raw
|
||||
}
|
||||
|
||||
func updateCRDT2PSetHeaders(header map[string]string, versions *objectVersions, versioningEnabled bool) []*oid.ID {
|
||||
if !versioningEnabled {
|
||||
header[versionsUnversionedAttr] = "true"
|
||||
|
|
@ -483,11 +418,17 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
|||
}
|
||||
|
||||
// objectDelete puts tombstone object into neofs.
|
||||
func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *oid.ID) error {
|
||||
addr := newAddress(cid, oid)
|
||||
n.objCache.Delete(addr)
|
||||
err := n.pool.DeleteObject(ctx, *addr, n.CallOptions(ctx)...)
|
||||
return n.transformNeofsError(ctx, err)
|
||||
func (n *layer) objectDelete(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) error {
|
||||
prm := PrmObjectDelete{
|
||||
Container: *idCnr,
|
||||
Object: *idObj,
|
||||
}
|
||||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth)
|
||||
|
||||
n.objCache.Delete(newAddress(idCnr, idObj))
|
||||
|
||||
return n.transformNeofsError(ctx, n.neoFS.DeleteObject(ctx, prm))
|
||||
}
|
||||
|
||||
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
||||
|
|
@ -737,7 +678,7 @@ func (n *layer) transformNeofsError(ctx context.Context, err error) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") {
|
||||
if errors.Is(err, ErrAccessDenied) {
|
||||
n.log.Debug("error was transformed", zap.String("request_id", api.GetRequestID(ctx)), zap.Error(err))
|
||||
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ package layer
|
|||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
|
|
@ -45,8 +43,8 @@ func (n *layer) headSystemObject(ctx context.Context, bkt *data.BucketInfo, objN
|
|||
|
||||
func (n *layer) deleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
|
||||
f := &findParams{
|
||||
filters: []filter{{attr: objectSystemAttributeName, val: name}},
|
||||
cid: bktInfo.CID,
|
||||
attr: [2]string{objectSystemAttributeName, name},
|
||||
cid: bktInfo.CID,
|
||||
}
|
||||
ids, err := n.objectSearch(ctx, f)
|
||||
if err != nil {
|
||||
|
|
@ -68,52 +66,41 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject
|
|||
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
|
||||
return nil, err
|
||||
}
|
||||
idsToDeleteArr := updateCRDT2PSetHeaders(p.Metadata, versions, false) // false means "last write wins"
|
||||
|
||||
attributes := make([]*object.Attribute, 0, 3)
|
||||
prm := PrmObjectCreate{
|
||||
Container: *p.BktInfo.CID,
|
||||
Creator: *p.BktInfo.Owner,
|
||||
Attributes: make([][2]string, 2, 2+len(p.Metadata)),
|
||||
Payload: p.Reader,
|
||||
}
|
||||
|
||||
filename := object.NewAttribute()
|
||||
filename.SetKey(objectSystemAttributeName)
|
||||
filename.SetValue(p.ObjName)
|
||||
|
||||
createdAt := object.NewAttribute()
|
||||
createdAt.SetKey(object.AttributeTimestamp)
|
||||
createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
|
||||
|
||||
versioningIgnore := object.NewAttribute()
|
||||
versioningIgnore.SetKey(attrVersionsIgnore)
|
||||
versioningIgnore.SetValue(strconv.FormatBool(true))
|
||||
|
||||
attributes = append(attributes, filename, createdAt, versioningIgnore)
|
||||
prm.Attributes[0][0], prm.Attributes[0][1] = objectSystemAttributeName, p.ObjName
|
||||
prm.Attributes[1][0], prm.Attributes[1][1] = attrVersionsIgnore, "true"
|
||||
|
||||
for k, v := range p.Metadata {
|
||||
attr := object.NewAttribute()
|
||||
if !IsSystemHeader(k) {
|
||||
k = p.Prefix + k
|
||||
}
|
||||
attr.SetKey(k)
|
||||
if p.Prefix == tagPrefix && v == "" {
|
||||
|
||||
if v == "" && p.Prefix == tagPrefix {
|
||||
v = tagEmptyMark
|
||||
}
|
||||
attr.SetValue(v)
|
||||
attributes = append(attributes, attr)
|
||||
|
||||
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
||||
}
|
||||
|
||||
raw := object.NewRaw()
|
||||
raw.SetOwnerID(p.BktInfo.Owner)
|
||||
raw.SetContainerID(p.BktInfo.CID)
|
||||
raw.SetAttributes(attributes...)
|
||||
|
||||
oid, err := n.pool.PutObject(ctx, *raw.Object(), p.Reader, n.CallOptions(ctx)...)
|
||||
id, err := n.neoFS.CreateObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, n.transformNeofsError(ctx, err)
|
||||
}
|
||||
|
||||
meta, err := n.objectHead(ctx, p.BktInfo.CID, oid)
|
||||
meta, err := n.objectHead(ctx, p.BktInfo.CID, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
idsToDeleteArr := updateCRDT2PSetHeaders(p.Metadata, versions, false) // false means "last write wins"
|
||||
|
||||
for _, id := range idsToDeleteArr {
|
||||
if err = n.objectDelete(ctx, p.BktInfo.CID, id); err != nil {
|
||||
n.log.Warn("couldn't delete system object",
|
||||
|
|
@ -178,8 +165,8 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName strin
|
|||
|
||||
func (n *layer) headSystemVersions(ctx context.Context, bkt *data.BucketInfo, sysName string) (*objectVersions, error) {
|
||||
f := &findParams{
|
||||
filters: []filter{{attr: objectSystemAttributeName, val: sysName}},
|
||||
cid: bkt.CID,
|
||||
attr: [2]string{objectSystemAttributeName, sysName},
|
||||
cid: bkt.CID,
|
||||
}
|
||||
ids, err := n.objectSearch(ctx, f)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -15,47 +15,168 @@ import (
|
|||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/accounting"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/logger"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object/id/test"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/owner"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/token"
|
||||
tokentest "github.com/nspcc-dev/neofs-sdk-go/token/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type testPool struct {
|
||||
pool.Pool
|
||||
type testNeoFS struct {
|
||||
NeoFS
|
||||
|
||||
objects map[string]*object.Object
|
||||
containers map[string]*container.Container
|
||||
currentEpoch uint64
|
||||
}
|
||||
|
||||
func newTestPool() *testPool {
|
||||
return &testPool{
|
||||
objects: make(map[string]*object.Object),
|
||||
containers: make(map[string]*container.Container),
|
||||
func (t *testNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (*cid.ID, error) {
|
||||
var opts []container.Option
|
||||
|
||||
opts = append(opts,
|
||||
container.WithOwnerID(&prm.Creator),
|
||||
container.WithPolicy(&prm.Policy),
|
||||
container.WithCustomBasicACL(prm.BasicACL),
|
||||
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(prm.Time.Unix(), 10)),
|
||||
)
|
||||
|
||||
if prm.Name != "" {
|
||||
opts = append(opts, container.WithAttribute(container.AttributeName, prm.Name))
|
||||
}
|
||||
|
||||
cnr := container.New(opts...)
|
||||
cnr.SetSessionToken(prm.SessionToken)
|
||||
|
||||
if prm.Name != "" {
|
||||
container.SetNativeName(cnr, prm.Name)
|
||||
}
|
||||
|
||||
b := make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := cid.New()
|
||||
id.SetSHA256(sha256.Sum256(b))
|
||||
t.containers[id.String()] = cnr
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (t *testPool) PutObject(_ context.Context, hdr object.Object, payload io.Reader, _ ...pool.CallOption) (*oid.ID, error) {
|
||||
func (t *testNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
|
||||
for k, v := range t.containers {
|
||||
if k == id.String() {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("container not found " + id.String())
|
||||
}
|
||||
|
||||
func (t *testNeoFS) UserContainers(_ context.Context, _ owner.ID) ([]cid.ID, error) {
|
||||
var res []cid.ID
|
||||
for k := range t.containers {
|
||||
var idCnr cid.ID
|
||||
if err := idCnr.Parse(k); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res = append(res, idCnr)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (t *testNeoFS) SelectObjects(_ context.Context, prm PrmObjectSelect) ([]oid.ID, error) {
|
||||
var filters object.SearchFilters
|
||||
filters.AddRootFilter()
|
||||
|
||||
if prm.FilePrefix != "" {
|
||||
filters.AddFilter(object.AttributeFileName, prm.FilePrefix, object.MatchCommonPrefix)
|
||||
}
|
||||
|
||||
if prm.ExactAttribute[0] != "" {
|
||||
filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual)
|
||||
}
|
||||
|
||||
cidStr := prm.Container.String()
|
||||
|
||||
var res []oid.ID
|
||||
|
||||
if len(filters) == 1 {
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) {
|
||||
res = append(res, *v.ID())
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
filter := filters[1]
|
||||
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual ||
|
||||
(filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) {
|
||||
return nil, fmt.Errorf("usupported filters")
|
||||
}
|
||||
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
|
||||
res = append(res, *v.ID())
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (t *testNeoFS) ReadObject(_ context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||
var addr address.Address
|
||||
addr.SetContainerID(&prm.Container)
|
||||
addr.SetObjectID(&prm.Object)
|
||||
|
||||
sAddr := addr.String()
|
||||
|
||||
if obj, ok := t.objects[sAddr]; ok {
|
||||
return &ObjectPart{
|
||||
Head: obj,
|
||||
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("object not found " + addr.String())
|
||||
}
|
||||
|
||||
func (t *testNeoFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*oid.ID, error) {
|
||||
id := test.ID()
|
||||
|
||||
raw := object.NewRawFrom(&hdr)
|
||||
attrs := make([]*object.Attribute, 0)
|
||||
|
||||
if prm.Filename != "" {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey(object.AttributeFileName)
|
||||
a.SetValue(prm.Filename)
|
||||
attrs = append(attrs, a)
|
||||
}
|
||||
|
||||
for i := range prm.Attributes {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey(prm.Attributes[i][0])
|
||||
a.SetValue(prm.Attributes[i][1])
|
||||
attrs = append(attrs, a)
|
||||
}
|
||||
|
||||
raw := object.NewRaw()
|
||||
raw.SetContainerID(&prm.Container)
|
||||
raw.SetID(id)
|
||||
raw.SetPayloadSize(prm.PayloadSize)
|
||||
raw.SetAttributes(attrs...)
|
||||
raw.SetCreationEpoch(t.currentEpoch)
|
||||
t.currentEpoch++
|
||||
|
||||
if payload != nil {
|
||||
all, err := io.ReadAll(payload)
|
||||
if prm.Payload != nil {
|
||||
all, err := io.ReadAll(prm.Payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -67,60 +188,21 @@ func (t *testPool) PutObject(_ context.Context, hdr object.Object, payload io.Re
|
|||
return raw.ID(), nil
|
||||
}
|
||||
|
||||
func (t *testPool) DeleteObject(ctx context.Context, addr address.Address, option ...pool.CallOption) error {
|
||||
func (t *testNeoFS) DeleteObject(_ context.Context, prm PrmObjectDelete) error {
|
||||
var addr address.Address
|
||||
addr.SetContainerID(&prm.Container)
|
||||
addr.SetObjectID(&prm.Object)
|
||||
|
||||
delete(t.objects, addr.String())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testPool) GetObject(_ context.Context, addr address.Address, _ ...pool.CallOption) (*pool.ResGetObject, error) {
|
||||
sAddr := addr.String()
|
||||
|
||||
if obj, ok := t.objects[sAddr]; ok {
|
||||
return &pool.ResGetObject{
|
||||
Header: *obj,
|
||||
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
||||
}, nil
|
||||
func newTestPool() *testNeoFS {
|
||||
return &testNeoFS{
|
||||
objects: make(map[string]*object.Object),
|
||||
containers: make(map[string]*container.Container),
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("object not found " + addr.String())
|
||||
}
|
||||
|
||||
func (t *testPool) HeadObject(ctx context.Context, addr address.Address, _ ...pool.CallOption) (*object.Object, error) {
|
||||
res, err := t.GetObject(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &res.Header, nil
|
||||
}
|
||||
|
||||
func (t *testPool) SearchObjects(_ context.Context, idCnr cid.ID, filters object.SearchFilters, _ ...pool.CallOption) (*pool.ResObjectSearch, error) {
|
||||
cidStr := idCnr.String()
|
||||
|
||||
var res []*oid.ID
|
||||
|
||||
if len(filters) == 1 {
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) {
|
||||
res = append(res, v.ID())
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
filter := filters[1]
|
||||
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual ||
|
||||
(filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) {
|
||||
return nil, fmt.Errorf("usupported filters")
|
||||
}
|
||||
|
||||
for k, v := range t.objects {
|
||||
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
|
||||
res = append(res, v.ID())
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool {
|
||||
|
|
@ -133,79 +215,6 @@ func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool
|
|||
return false
|
||||
}
|
||||
|
||||
func (t *testPool) PutContainer(ctx context.Context, container *container.Container, option ...pool.CallOption) (*cid.ID, error) {
|
||||
b := make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := cid.New()
|
||||
id.SetSHA256(sha256.Sum256(b))
|
||||
t.containers[id.String()] = container
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (t *testPool) GetContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*container.Container, error) {
|
||||
for k, v := range t.containers {
|
||||
if k == id.String() {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("container not found " + id.String())
|
||||
}
|
||||
|
||||
func (t *testPool) ListContainers(ctx context.Context, id *owner.ID, option ...pool.CallOption) ([]*cid.ID, error) {
|
||||
var res []*cid.ID
|
||||
for k := range t.containers {
|
||||
cID := cid.New()
|
||||
if err := cID.Parse(k); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res = append(res, cID)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (t *testPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) error {
|
||||
delete(t.containers, id.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testPool) GetEACL(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*eacl.Table, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) Balance(ctx context.Context, owner *owner.ID, opts ...pool.CallOption) (*accounting.Decimal, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) SetEACL(ctx context.Context, table *eacl.Table, option ...pool.CallOption) error {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...pool.CallOption) error {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) Connection() (pool.Client, *session.Token, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) Close() {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *testPool) OwnerID() *owner.ID {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
|
||||
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
|
||||
Bucket: tc.bktID.String(),
|
||||
|
|
@ -298,7 +307,7 @@ func (tc *testContext) checkListObjects(ids ...*oid.ID) {
|
|||
}
|
||||
|
||||
func (tc *testContext) getSystemObject(objectName string) *object.Object {
|
||||
for _, obj := range tc.testPool.objects {
|
||||
for _, obj := range tc.testNeoFS.objects {
|
||||
for _, attr := range obj.Attributes() {
|
||||
if attr.Key() == objectSystemAttributeName && attr.Value() == objectName {
|
||||
return obj
|
||||
|
|
@ -309,22 +318,25 @@ func (tc *testContext) getSystemObject(objectName string) *object.Object {
|
|||
}
|
||||
|
||||
type testContext struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
layer Client
|
||||
bkt string
|
||||
bktID *cid.ID
|
||||
obj string
|
||||
testPool *testPool
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
layer Client
|
||||
bkt string
|
||||
bktID *cid.ID
|
||||
obj string
|
||||
testNeoFS *testNeoFS
|
||||
}
|
||||
|
||||
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
bearerToken := tokentest.BearerToken()
|
||||
require.NoError(t, bearerToken.SignToken(&key.PrivateKey))
|
||||
|
||||
ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
BearerToken: token.NewBearerToken(),
|
||||
BearerToken: bearerToken,
|
||||
GateKey: key.PublicKey(),
|
||||
},
|
||||
})
|
||||
|
|
@ -333,8 +345,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
tp := newTestPool()
|
||||
|
||||
bktName := "testbucket1"
|
||||
cnr := container.New(container.WithAttribute(container.AttributeName, bktName))
|
||||
bktID, err := tp.PutContainer(ctx, cnr)
|
||||
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
|
||||
Name: bktName,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
config := DefaultCachesConfigs()
|
||||
|
|
@ -348,19 +361,17 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
}
|
||||
|
||||
return &testContext{
|
||||
ctx: ctx,
|
||||
layer: NewLayer(l, tp, layerCfg),
|
||||
bkt: bktName,
|
||||
bktID: bktID,
|
||||
obj: "obj1",
|
||||
t: t,
|
||||
testPool: tp,
|
||||
ctx: ctx,
|
||||
layer: NewLayer(l, tp, layerCfg),
|
||||
bkt: bktName,
|
||||
bktID: bktID,
|
||||
obj: "obj1",
|
||||
t: t,
|
||||
testNeoFS: tp,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimpleVersioning(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
tc := prepareContext(t)
|
||||
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
|
||||
Bucket: tc.bktID.String(),
|
||||
|
|
@ -385,8 +396,6 @@ func TestSimpleVersioning(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestSimpleNoVersioning(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
tc := prepareContext(t)
|
||||
|
||||
obj1Content1 := []byte("content obj1 v1")
|
||||
|
|
@ -404,8 +413,6 @@ func TestSimpleNoVersioning(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestVersioningDeleteObject(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
tc := prepareContext(t)
|
||||
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
|
||||
Bucket: tc.bktID.String(),
|
||||
|
|
@ -423,8 +430,6 @@ func TestVersioningDeleteObject(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
tc := prepareContext(t)
|
||||
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
|
||||
Bucket: tc.bktID.String(),
|
||||
|
|
@ -532,8 +537,6 @@ func TestGetLastVersion(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestNoVersioningDeleteObject(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
tc := prepareContext(t)
|
||||
|
||||
tc.putObject([]byte("content obj1 v1"))
|
||||
|
|
@ -789,8 +792,6 @@ func TestUpdateCRDT2PSetHeaders(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestSystemObjectsVersioning(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
cacheConfig := DefaultCachesConfigs()
|
||||
cacheConfig.System.Lifetime = 0
|
||||
|
||||
|
|
@ -801,7 +802,7 @@ func TestSystemObjectsVersioning(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
objMeta, ok := tc.testPool.objects[objInfo.Address().String()]
|
||||
objMeta, ok := tc.testNeoFS.objects[objInfo.Address().String()]
|
||||
require.True(t, ok)
|
||||
|
||||
_, err = tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
|
||||
|
|
@ -811,7 +812,7 @@ func TestSystemObjectsVersioning(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
// simulate failed deletion
|
||||
tc.testPool.objects[objInfo.Address().String()] = objMeta
|
||||
tc.testNeoFS.objects[objInfo.Address().String()] = objMeta
|
||||
|
||||
versioning, err := tc.layer.GetBucketVersioning(tc.ctx, tc.bkt)
|
||||
require.NoError(t, err)
|
||||
|
|
@ -819,8 +820,6 @@ func TestSystemObjectsVersioning(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDeleteSystemObjectsVersioning(t *testing.T) {
|
||||
// https://github.com/nspcc-dev/neofs-s3-gw/issues/349
|
||||
t.Skip("pool.Pool does not support overriding")
|
||||
cacheConfig := DefaultCachesConfigs()
|
||||
cacheConfig.System.Lifetime = 0
|
||||
|
||||
|
|
@ -840,7 +839,7 @@ func TestDeleteSystemObjectsVersioning(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
// simulate failed deletion
|
||||
tc.testPool.objects[newAddress(objMeta.ContainerID(), objMeta.ID()).String()] = objMeta
|
||||
tc.testNeoFS.objects[newAddress(objMeta.ContainerID(), objMeta.ID()).String()] = objMeta
|
||||
|
||||
tagging, err := tc.layer.GetBucketTagging(tc.ctx, tc.bkt)
|
||||
require.NoError(t, err)
|
||||
|
|
|
|||
|
|
@ -5,24 +5,27 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neo-go/pkg/rpc/client"
|
||||
neofsclient "github.com/nspcc-dev/neofs-sdk-go/client"
|
||||
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/resolver"
|
||||
)
|
||||
|
||||
const (
|
||||
NNSResolver = "nns"
|
||||
DNSResolver = "dns"
|
||||
|
||||
networkSystemDNSParam = "SystemDNS"
|
||||
)
|
||||
|
||||
// NeoFS represents virtual connection to the NeoFS network.
|
||||
type NeoFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the NeoFS.
|
||||
//
|
||||
// Returns exactly on non-zero value. Returns any error encountered
|
||||
// which prevented the parameter to be read.
|
||||
SystemDNS(context.Context) (string, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Pool pool.Pool
|
||||
RPC *client.Client
|
||||
NeoFS NeoFS
|
||||
RPC *client.Client
|
||||
}
|
||||
|
||||
type BucketResolver struct {
|
||||
|
|
@ -69,7 +72,7 @@ func NewResolver(order []string, cfg *Config) (*BucketResolver, error) {
|
|||
func newResolver(name string, cfg *Config, next *BucketResolver) (*BucketResolver, error) {
|
||||
switch name {
|
||||
case DNSResolver:
|
||||
return NewDNSResolver(cfg.Pool, next)
|
||||
return NewDNSResolver(cfg.NeoFS, next)
|
||||
case NNSResolver:
|
||||
return NewNNSResolver(cfg.RPC, next)
|
||||
default:
|
||||
|
|
@ -77,38 +80,15 @@ func newResolver(name string, cfg *Config, next *BucketResolver) (*BucketResolve
|
|||
}
|
||||
}
|
||||
|
||||
func NewDNSResolver(p pool.Pool, next *BucketResolver) (*BucketResolver, error) {
|
||||
if p == nil {
|
||||
func NewDNSResolver(neoFS NeoFS, next *BucketResolver) (*BucketResolver, error) {
|
||||
if neoFS == nil {
|
||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||
}
|
||||
|
||||
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
|
||||
conn, _, err := p.Connection()
|
||||
domain, err := neoFS.SystemDNS(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
networkInfoRes, err := conn.NetworkInfo(ctx, neofsclient.PrmNetworkInfo{})
|
||||
if err == nil {
|
||||
err = apistatus.ErrFromStatus(networkInfoRes.Status())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
networkInfo := networkInfoRes.Info()
|
||||
|
||||
var domain string
|
||||
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
|
||||
if string(parameter.Key()) == networkSystemDNSParam {
|
||||
domain = string(parameter.Value())
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
if domain == "" {
|
||||
return nil, fmt.Errorf("couldn't resolve container '%s': not found", name)
|
||||
return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
||||
}
|
||||
|
||||
domain = name + "." + domain
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue