From 13080d6d969a8e858a73f7bfac53327bf3b08b4f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 4 Mar 2022 16:07:27 +0300 Subject: [PATCH] [#195] Refactor TestNeoFS Signed-off-by: Denis Kirillov --- api/handler/handlers_test.go | 49 +++--- api/layer/container.go | 32 ++-- api/layer/layer.go | 216 +------------------------- api/layer/neofs/neofs.go | 223 +++++++++++++++++++++++++++ api/layer/object.go | 17 ++- api/layer/system_object.go | 23 +-- api/layer/versioning_test.go | 24 +-- api/mock/sdk_pool_mock.go | 216 -------------------------- cmd/s3-gw/neofs.go | 16 +- internal/neofs/neofs.go | 51 +++---- internal/neofstest/neofs_mock.go | 251 +++++++++++++++++++++++++++++++ 11 files changed, 585 insertions(+), 533 deletions(-) create mode 100644 api/layer/neofs/neofs.go delete mode 100644 api/mock/sdk_pool_mock.go create mode 100644 internal/neofstest/neofs_mock.go diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 3137652..f489296 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -4,38 +4,34 @@ import ( "bytes" "context" "encoding/xml" - "fmt" "math/rand" "net/http" "net/http/httptest" - "strconv" "testing" - "github.com/nspcc-dev/neofs-s3-gw/api" - - "github.com/nspcc-dev/neofs-s3-gw/api/data" - - "github.com/nspcc-dev/neofs-s3-gw/api/resolver" - "github.com/nspcc-dev/neofs-sdk-go/container" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/layer" - "github.com/nspcc-dev/neofs-s3-gw/api/mock" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" + "github.com/nspcc-dev/neofs-s3-gw/api/resolver" + "github.com/nspcc-dev/neofs-s3-gw/internal/neofstest" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/logger" + "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/stretchr/testify/require" ) type handlerContext struct { h *handler - tp *mock.TestPool + tp *neofstest.TestNeoFS } func (hc *handlerContext) Handler() *handler { return hc.h } -func (hc *handlerContext) MockedPool() *mock.TestPool { +func (hc *handlerContext) MockedPool() *neofstest.TestNeoFS { return hc.tp } @@ -49,19 +45,11 @@ func prepareHandlerContext(t *testing.T) *handlerContext { l, err := logger.New(logger.WithTraceLevel("panic")) require.NoError(t, err) - tp := mock.NewTestPool() + tp := neofstest.NewTestNeoFS() testResolver := &resolver.BucketResolver{Name: "test_resolver"} - testResolver.SetResolveFunc(func(ctx context.Context, name string) (*cid.ID, error) { - for id, cnr := range tp.Containers { - for _, attr := range cnr.Attributes() { - if attr.Key() == container.AttributeName && attr.Value() == name { - cnrID := cid.New() - return cnrID, cnrID.Parse(id) - } - } - } - return nil, fmt.Errorf("couldn't resolve container name") + testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) { + return tp.ContainerID(name) }) layerCfg := &layer.Config{ @@ -83,15 +71,17 @@ func prepareHandlerContext(t *testing.T) *handlerContext { } func createTestBucket(ctx context.Context, t *testing.T, h *handlerContext, bktName string) { - cnr := container.New(container.WithAttribute(container.AttributeName, bktName)) - _, err := h.MockedPool().PutContainer(ctx, cnr) + _, err := h.MockedPool().CreateContainer(ctx, neofs.PrmContainerCreate{ + Name: bktName, + }) require.NoError(t, err) } func createTestBucketWithLock(ctx context.Context, t *testing.T, h *handlerContext, bktName string, conf *data.ObjectLockConfiguration) { - cnr := container.New(container.WithAttribute(container.AttributeName, bktName), - container.WithAttribute(layer.AttributeLockEnabled, strconv.FormatBool(true))) - cnrID, err := h.MockedPool().PutContainer(ctx, cnr) + cnrID, err := h.MockedPool().CreateContainer(ctx, neofs.PrmContainerCreate{ + Name: bktName, + AdditionalAttributes: [][2]string{{layer.AttributeLockEnabled, "true"}}, + }) require.NoError(t, err) sp := &layer.PutSettingsParams{ @@ -99,6 +89,7 @@ func createTestBucketWithLock(ctx context.Context, t *testing.T, h *handlerConte CID: cnrID, Name: bktName, ObjectLockEnabled: true, + Owner: owner.NewID(), }, Settings: &data.BucketSettings{ VersioningEnabled: true, diff --git a/api/layer/container.go b/api/layer/container.go index d675ab1..89af0ea 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-sdk-go/acl" "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" @@ -131,23 +132,28 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*ci LocationConstraint: p.LocationConstraint, } - var locConstAttr *container.Attribute + var attributes [][2]string if p.LocationConstraint != "" { - locConstAttr = container.NewAttribute() - locConstAttr.SetKey(attributeLocationConstraint) - locConstAttr.SetValue(p.LocationConstraint) + attributes = append(attributes, [2]string{ + attributeLocationConstraint, p.LocationConstraint, + }) } - //todo add lock enabled attr - if bktInfo.CID, err = n.neoFS.CreateContainer(ctx, PrmContainerCreate{ - Creator: *bktInfo.Owner, - Policy: *p.Policy, - Name: p.Name, - SessionToken: p.SessionToken, - Time: bktInfo.Created, - BasicACL: acl.BasicACL(p.ACL), - LocationConstraintAttribute: locConstAttr, + if p.ObjectLockEnabled { + attributes = append(attributes, [2]string{ + AttributeLockEnabled, "true", + }) + } + + if bktInfo.CID, err = n.neoFS.CreateContainer(ctx, neofs.PrmContainerCreate{ + Creator: *bktInfo.Owner, + Policy: *p.Policy, + Name: p.Name, + SessionToken: p.SessionToken, + Time: bktInfo.Created, + BasicACL: acl.BasicACL(p.ACL), + AdditionalAttributes: attributes, }); err != nil { return nil, err } diff --git a/api/layer/layer.go b/api/layer/layer.go index b087e0d..a257a58 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -4,241 +4,33 @@ import ( "bytes" "context" "crypto/ecdsa" - stderrors "errors" "fmt" "io" "net/url" "strings" - "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-s3-gw/api/notifications" "github.com/nspcc-dev/neofs-s3-gw/api/resolver" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" - "github.com/nspcc-dev/neofs-sdk-go/acl" - "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/eacl" "github.com/nspcc-dev/neofs-sdk-go/netmap" - "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/session" - "github.com/nspcc-dev/neofs-sdk-go/token" "go.uber.org/zap" ) -// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation. -type PrmContainerCreate struct { - // NeoFS identifier of the container creator. - Creator owner.ID - - // Container placement policy. - Policy netmap.PlacementPolicy - - // Name for the container. - Name string - - // Token of the container's creation session. Nil means session absence. - SessionToken *session.Token - - // Time when container is created. - Time time.Time - - // Basic ACL of the container. - BasicACL acl.BasicACL - - // Attribute for LocationConstraint parameter (optional). - LocationConstraintAttribute *container.Attribute -} - -// PrmAuth groups authentication parameters for the NeoFS operation. -type PrmAuth struct { - // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. - BearerToken *token.BearerToken - - // Private key used for the operation if BearerToken is missing (in this case non-nil). - PrivateKey *ecdsa.PrivateKey -} - -// PrmObjectSelect groups parameters of NeoFS.SelectObjects operation. -type PrmObjectSelect struct { - // Authentication parameters. - PrmAuth - - // Container to select the objects from. - Container cid.ID - - // Key-value object attribute which should exactly be - // presented in selected objects. Optional, empty key means any. - ExactAttribute [2]string - - // File prefix of the selected objects. Optional, empty value means any. - FilePrefix string -} - -// PrmObjectRead groups parameters of NeoFS.ReadObject operation. -type PrmObjectRead struct { - // Authentication parameters. - PrmAuth - - // Container to read the object header from. - Container cid.ID - - // ID of the object for which to read the header. - Object oid.ID - - // Flag to read object header. - WithHeader bool - - // Flag to read object payload. False overlaps payload range. - WithPayload bool - - // Offset-length range of the object payload to be read. - PayloadRange [2]uint64 -} - -// ObjectPart represents partially read NeoFS object. -type ObjectPart struct { - // Object header with optional in-memory payload part. - Head *object.Object - - // Object payload part encapsulated in io.Reader primitive. - // Returns ErrAccessDenied on read access violation. - Payload io.ReadCloser -} - -// PrmObjectCreate groups parameters of NeoFS.CreateObject operation. -type PrmObjectCreate struct { - // Authentication parameters. - PrmAuth - - // Container to store the object. - Container cid.ID - - // NeoFS identifier of the object creator. - Creator owner.ID - - // Key-value object attributes. - Attributes [][2]string - - // Full payload size (optional). - PayloadSize uint64 - - // Associated filename (optional). - Filename string - - // Object payload encapsulated in io.Reader primitive. - Payload io.Reader -} - -// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation. -type PrmObjectDelete struct { - // Authentication parameters. - PrmAuth - - // Container to delete the object from. - Container cid.ID - - // Identifier of the removed object. - Object oid.ID -} - -// ErrAccessDenied is returned from NeoFS in case of access violation. -var ErrAccessDenied = stderrors.New("access denied") - -// NeoFS represents virtual connection to NeoFS network. -type NeoFS interface { - // CreateContainer creates and saves parameterized container in NeoFS. - // Returns ID of the saved container. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the container to be created. - CreateContainer(context.Context, PrmContainerCreate) (*cid.ID, error) - - // Container reads container from NeoFS by ID. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the container to be read. - Container(context.Context, cid.ID) (*container.Container, error) - - // UserContainers reads list of the containers owned by specified user. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the containers to be listed. - UserContainers(context.Context, owner.ID) ([]cid.ID, error) - - // SetContainerEACL saves eACL table of the container in NeoFS. - // - // Returns any error encountered which prevented the eACL to be saved. - SetContainerEACL(context.Context, eacl.Table) error - - // ContainerEACL reads container eACL from NeoFS by container ID. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the eACL to be read. - ContainerEACL(context.Context, cid.ID) (*eacl.Table, error) - - // DeleteContainer marks the container to be removed from NeoFS by ID. - // Request is sent within session if the session token is specified. - // Successful return does not guarantee the actual removal. - // - // Returns any error encountered which prevented the removal request to be sent. - DeleteContainer(context.Context, cid.ID, *session.Token) error - - // SelectObjects perform object selection from the NeoFS container according - // to specified parameters. Selects user objects only. - // - // Returns ErrAccessDenied on selection access violation. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the objects to be selected. - SelectObjects(context.Context, PrmObjectSelect) ([]oid.ID, error) - - // ReadObject reads part of the object from the NeoFS container by identifier. - // Exact part is returned according to the parameters: - // * with header only: empty payload (both in-mem and reader parts are nil); - // * with payload only: header is nil (zero range means full payload); - // * with header and payload: full in-mem object, payload reader is nil. - // - // WithHeader or WithPayload is true. Range length is positive if offset is positive. - // - // Payload reader should be closed if it is no longer needed. - // - // Returns ErrAccessDenied on read access violation. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the object header to be read. - ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error) - - // CreateObject creates and saves parameterized object in the NeoFS container. - // Returns ID of the saved object. - // - // Creation time should be written into object (UTC). - // - // Returns ErrAccessDenied on write access violation. - // - // Returns exactly one non-nil value. Returns any error encountered which - // prevented the container to be created. - CreateObject(context.Context, PrmObjectCreate) (*oid.ID, error) - - // DeleteObject marks the object to be removed from the NeoFS container by identifier. - // Successful return does not guarantee the actual removal. - // - // Returns ErrAccessDenied on remove access violation. - // - // Returns any error encountered which prevented the removal request to be sent. - DeleteObject(context.Context, PrmObjectDelete) error -} - type ( layer struct { - neoFS NeoFS + neoFS neofs.NeoFS log *zap.Logger anonKey AnonymousKey resolver *resolver.BucketResolver @@ -455,7 +247,7 @@ func DefaultCachesConfigs() *CachesConfig { // NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. -func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client { +func NewLayer(log *zap.Logger, neoFS neofs.NeoFS, config *Config) Client { return &layer{ neoFS: neoFS, log: log, @@ -493,7 +285,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID { return owner.NewIDFromPublicKey((*ecdsa.PublicKey)(n.EphemeralKey())) } -func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth) { +func (n *layer) prepareAuthParameters(ctx context.Context, prm *neofs.PrmAuth) { if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil { prm.BearerToken = bd.Gate.BearerToken return diff --git a/api/layer/neofs/neofs.go b/api/layer/neofs/neofs.go new file mode 100644 index 0000000..595f201 --- /dev/null +++ b/api/layer/neofs/neofs.go @@ -0,0 +1,223 @@ +package neofs + +import ( + "context" + "crypto/ecdsa" + "errors" + "io" + "time" + + "github.com/nspcc-dev/neofs-sdk-go/acl" + "github.com/nspcc-dev/neofs-sdk-go/container" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/eacl" + "github.com/nspcc-dev/neofs-sdk-go/netmap" + "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "github.com/nspcc-dev/neofs-sdk-go/owner" + "github.com/nspcc-dev/neofs-sdk-go/session" + "github.com/nspcc-dev/neofs-sdk-go/token" +) + +// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation. +type PrmContainerCreate struct { + // NeoFS identifier of the container creator. + Creator owner.ID + + // Container placement policy. + Policy netmap.PlacementPolicy + + // Name for the container. + Name string + + // Token of the container's creation session. Nil means session absence. + SessionToken *session.Token + + // Time when container is created. + Time time.Time + + // Basic ACL of the container. + BasicACL acl.BasicACL + + // Attributes for optional parameters. + AdditionalAttributes [][2]string +} + +// PrmAuth groups authentication parameters for the NeoFS operation. +type PrmAuth struct { + // Bearer token to be used for the operation. Overlaps PrivateKey. Optional. + BearerToken *token.BearerToken + + // Private key used for the operation if BearerToken is missing (in this case non-nil). + PrivateKey *ecdsa.PrivateKey +} + +// PrmObjectSelect groups parameters of NeoFS.SelectObjects operation. +type PrmObjectSelect struct { + // Authentication parameters. + PrmAuth + + // Container to select the objects from. + Container cid.ID + + // Key-value object attribute which should exactly be + // presented in selected objects. Optional, empty key means any. + ExactAttribute [2]string + + // File prefix of the selected objects. Optional, empty value means any. + FilePrefix string +} + +// PrmObjectRead groups parameters of NeoFS.ReadObject operation. +type PrmObjectRead struct { + // Authentication parameters. + PrmAuth + + // Container to read the object header from. + Container cid.ID + + // ID of the object for which to read the header. + Object oid.ID + + // Flag to read object header. + WithHeader bool + + // Flag to read object payload. False overlaps payload range. + WithPayload bool + + // Offset-length range of the object payload to be read. + PayloadRange [2]uint64 +} + +// ObjectPart represents partially read NeoFS object. +type ObjectPart struct { + // Object header with optional in-memory payload part. + Head *object.Object + + // Object payload part encapsulated in io.Reader primitive. + // Returns ErrAccessDenied on read access violation. + Payload io.ReadCloser +} + +// PrmObjectCreate groups parameters of NeoFS.CreateObject operation. +type PrmObjectCreate struct { + // Authentication parameters. + PrmAuth + + // Container to store the object. + Container cid.ID + + // NeoFS identifier of the object creator. + Creator owner.ID + + // Key-value object attributes. + Attributes [][2]string + + // Full payload size (optional). + PayloadSize uint64 + + // Associated filename (optional). + Filename string + + // Object payload encapsulated in io.Reader primitive. + Payload io.Reader +} + +// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation. +type PrmObjectDelete struct { + // Authentication parameters. + PrmAuth + + // Container to delete the object from. + Container cid.ID + + // Identifier of the removed object. + Object oid.ID +} + +// ErrAccessDenied is returned from NeoFS in case of access violation. +var ErrAccessDenied = errors.New("access denied") + +// NeoFS represents virtual connection to NeoFS network. +type NeoFS interface { + // CreateContainer creates and saves parameterized container in NeoFS. + // Returns ID of the saved container. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the container to be created. + CreateContainer(context.Context, PrmContainerCreate) (*cid.ID, error) + + // Container reads container from NeoFS by ID. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the container to be read. + Container(context.Context, cid.ID) (*container.Container, error) + + // UserContainers reads list of the containers owned by specified user. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the containers to be listed. + UserContainers(context.Context, owner.ID) ([]cid.ID, error) + + // SetContainerEACL saves eACL table of the container in NeoFS. + // + // Returns any error encountered which prevented the eACL to be saved. + SetContainerEACL(context.Context, eacl.Table) error + + // ContainerEACL reads container eACL from NeoFS by container ID. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the eACL to be read. + ContainerEACL(context.Context, cid.ID) (*eacl.Table, error) + + // DeleteContainer marks the container to be removed from NeoFS by ID. + // Request is sent within session if the session token is specified. + // Successful return does not guarantee the actual removal. + // + // Returns any error encountered which prevented the removal request to be sent. + DeleteContainer(context.Context, cid.ID, *session.Token) error + + // SelectObjects perform object selection from the NeoFS container according + // to specified parameters. Selects user objects only. + // + // Returns ErrAccessDenied on selection access violation. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the objects to be selected. + SelectObjects(context.Context, PrmObjectSelect) ([]oid.ID, error) + + // ReadObject reads part of the object from the NeoFS container by identifier. + // Exact part is returned according to the parameters: + // * with header only: empty payload (both in-mem and reader parts are nil); + // * with payload only: header is nil (zero range means full payload); + // * with header and payload: full in-mem object, payload reader is nil. + // + // WithHeader or WithPayload is true. Range length is positive if offset is positive. + // + // Payload reader should be closed if it is no longer needed. + // + // Returns ErrAccessDenied on read access violation. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the object header to be read. + ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error) + + // CreateObject creates and saves parameterized object in the NeoFS container. + // Returns ID of the saved object. + // + // Creation time should be written into object (UTC). + // + // Returns ErrAccessDenied on write access violation. + // + // Returns exactly one non-nil value. Returns any error encountered which + // prevented the container to be created. + CreateObject(context.Context, PrmObjectCreate) (*oid.ID, error) + + // DeleteObject marks the object to be removed from the NeoFS container by identifier. + // Successful return does not guarantee the actual removal. + // + // Returns ErrAccessDenied on remove access violation. + // + // Returns any error encountered which prevented the removal request to be sent. + DeleteObject(context.Context, PrmObjectDelete) error +} diff --git a/api/layer/object.go b/api/layer/object.go index 62dfcd6..b7854c4 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/data" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" @@ -75,7 +76,7 @@ func (n *layer) objectSearchByName(ctx context.Context, cid *cid.ID, filename st // objectSearch returns all available objects by search params. func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]oid.ID, error) { - prm := PrmObjectSelect{ + prm := neofs.PrmObjectSelect{ Container: *p.cid, ExactAttribute: p.attr, FilePrefix: p.prefix, @@ -97,7 +98,7 @@ func newAddress(cid *cid.ID, oid *oid.ID) *address.Address { // objectHead returns all object's headers. func (n *layer) objectHead(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) (*object.Object, error) { - prm := PrmObjectRead{ + prm := neofs.PrmObjectRead{ Container: *idCnr, Object: *idObj, WithHeader: true, @@ -116,7 +117,7 @@ func (n *layer) objectHead(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) (* // initializes payload reader of the NeoFS object. // Zero range corresponds to full payload (panics if only offset is set). func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) { - prm := PrmObjectRead{ + prm := neofs.PrmObjectRead{ Container: *p.cid, Object: *p.oid, WithPayload: true, @@ -135,7 +136,7 @@ func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re // objectGet returns an object with payload in the object. func (n *layer) objectGet(ctx context.Context, addr *address.Address) (*object.Object, error) { - prm := PrmObjectRead{ + prm := neofs.PrmObjectRead{ Container: *addr.ContainerID(), Object: *addr.ObjectID(), WithHeader: true, @@ -174,7 +175,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec } } - prm := PrmObjectCreate{ + prm := neofs.PrmObjectCreate{ Container: *bkt.CID, Creator: *own, PayloadSize: uint64(p.Size), @@ -202,7 +203,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *data.BucketInfo, p *PutObjec } if p.Lock != nil { - objInfo := &data.ObjectInfo{ID: oid, Name: p.Object} + objInfo := &data.ObjectInfo{ID: id, Name: p.Object} if p.Lock.LegalHold { if err = n.putLockObject(ctx, bkt, objInfo.LegalHoldObject(), p.Lock); err != nil { return nil, err @@ -437,7 +438,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb // objectDelete puts tombstone object into neofs. func (n *layer) objectDelete(ctx context.Context, idCnr *cid.ID, idObj *oid.ID) error { - prm := PrmObjectDelete{ + prm := neofs.PrmObjectDelete{ Container: *idCnr, Object: *idObj, } @@ -696,7 +697,7 @@ func (n *layer) transformNeofsError(ctx context.Context, err error) error { return nil } - if errors.Is(err, ErrAccessDenied) { + if errors.Is(err, neofs.ErrAccessDenied) { n.log.Debug("error was transformed", zap.String("request_id", api.GetRequestID(ctx)), zap.Error(err)) return apiErrors.GetAPIError(apiErrors.ErrAccessDenied) } diff --git a/api/layer/system_object.go b/api/layer/system_object.go index c085ee3..70020f2 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -6,9 +6,12 @@ import ( "encoding/json" "encoding/xml" "fmt" + "strconv" + "time" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" "go.uber.org/zap" @@ -78,7 +81,7 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject idsToDeleteArr := updateCRDT2PSetHeaders(p.Metadata, versions, false) // false means "last write wins" // note that updateCRDT2PSetHeaders modifies p.Metadata and must be called further processing - prm := PrmObjectCreate{ + prm := neofs.PrmObjectCreate{ Container: *p.BktInfo.CID, Creator: *p.BktInfo.Owner, Attributes: make([][2]string, 2, 2+len(p.Metadata)), @@ -276,17 +279,19 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err return nil } -func attributesFromLock(lock *data.ObjectLock) []*object.Attribute { - var result []*object.Attribute +func attributesFromLock(lock *data.ObjectLock) [][2]string { + var result [][2]string if !lock.Until.IsZero() { - attrRetainUntil := object.NewAttribute() - attrRetainUntil.SetKey(AttributeRetainUntil) - attrRetainUntil.SetValue(lock.Until.Format(time.RFC3339)) + attrRetainUntil := [2]string{ + AttributeRetainUntil, + lock.Until.Format(time.RFC3339), + } result = append(result, attrRetainUntil) if lock.IsCompliance { - attrCompliance := object.NewAttribute() - attrCompliance.SetKey(AttributeComplianceMode) - attrCompliance.SetValue(strconv.FormatBool(true)) + attrCompliance := [2]string{ + AttributeComplianceMode, + strconv.FormatBool(true), + } result = append(result, attrCompliance) } } diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 331fc92..ed6cee3 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -10,14 +10,15 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" - "github.com/nspcc-dev/neofs-s3-gw/api/mock" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" + "github.com/nspcc-dev/neofs-s3-gw/internal/neofstest" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/logger" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/object/id/test" + "github.com/nspcc-dev/neofs-sdk-go/owner" tokentest "github.com/nspcc-dev/neofs-sdk-go/token/test" "github.com/stretchr/testify/require" ) @@ -114,7 +115,7 @@ func (tc *testContext) checkListObjects(ids ...*oid.ID) { } func (tc *testContext) getSystemObject(objectName string) *object.Object { - for _, obj := range tc.testNeoFS.Objects { + for _, obj := range tc.testNeoFS.Objects() { for _, attr := range obj.Attributes() { if attr.Key() == objectSystemAttributeName && attr.Value() == objectName { return obj @@ -132,7 +133,7 @@ type testContext struct { bktID *cid.ID bktInfo *data.BucketInfo obj string - testNeoFS *mock.TestNeoFS + testNeoFS *neofstest.TestNeoFS } func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { @@ -150,10 +151,10 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { }) l, err := logger.New(logger.WithTraceLevel("panic")) require.NoError(t, err) - tp := mock.NewTestPool() + tp := neofstest.NewTestNeoFS() bktName := "testbucket1" - bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{ + bktID, err := tp.CreateContainer(ctx, neofs.PrmContainerCreate{ Name: bktName, }) require.NoError(t, err) @@ -174,8 +175,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { bkt: bktName, bktID: bktID, bktInfo: &data.BucketInfo{ - Name: bktName, - CID: bktID, + Name: bktName, + CID: bktID, + Owner: owner.NewID(), }, obj: "obj1", t: t, @@ -623,12 +625,12 @@ func TestSystemObjectsVersioning(t *testing.T) { }) require.NoError(t, err) - addr := object.NewAddress() + addr := address.NewAddress() addr.SetContainerID(objMeta.ContainerID()) addr.SetObjectID(objMeta.ID()) // simulate failed deletion - tc.testNeoFS.Objects[addr.String()] = objMeta + tc.testNeoFS.AddObject(addr.String(), objMeta) bktInfo := &data.BucketInfo{ Name: tc.bkt, @@ -660,7 +662,7 @@ func TestDeleteSystemObjectsVersioning(t *testing.T) { require.NoError(t, err) // simulate failed deletion - tc.testNeoFS.Objects[newAddress(objMeta.ContainerID(), objMeta.ID()).String()] = objMeta + tc.testNeoFS.AddObject(newAddress(objMeta.ContainerID(), objMeta.ID()).String(), objMeta) tagging, err := tc.layer.GetBucketTagging(tc.ctx, tc.bkt) require.NoError(t, err) diff --git a/api/mock/sdk_pool_mock.go b/api/mock/sdk_pool_mock.go deleted file mode 100644 index 5598803..0000000 --- a/api/mock/sdk_pool_mock.go +++ /dev/null @@ -1,216 +0,0 @@ -package mock - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "fmt" - "io" - "strings" - - "github.com/nspcc-dev/neofs-sdk-go/accounting" - "github.com/nspcc-dev/neofs-sdk-go/client" - "github.com/nspcc-dev/neofs-sdk-go/container" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/eacl" - "github.com/nspcc-dev/neofs-sdk-go/object" - "github.com/nspcc-dev/neofs-sdk-go/owner" - "github.com/nspcc-dev/neofs-sdk-go/pool" - "github.com/nspcc-dev/neofs-sdk-go/session" -) - -type TestPool struct { - Objects map[string]*object.Object - Containers map[string]*container.Container - CurrentEpoch uint64 -} - -func NewTestPool() *TestPool { - return &TestPool{ - Objects: make(map[string]*object.Object), - Containers: make(map[string]*container.Container), - } -} - -func (t *TestPool) PutObject(ctx context.Context, params *client.PutObjectParams, option ...pool.CallOption) (*object.ID, error) { - b := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - return nil, err - } - - oid := object.NewID() - oid.SetSHA256(sha256.Sum256(b)) - - raw := object.NewRawFrom(params.Object()) - raw.SetID(oid) - raw.SetCreationEpoch(t.CurrentEpoch) - t.CurrentEpoch++ - - if params.PayloadReader() != nil { - all, err := io.ReadAll(params.PayloadReader()) - if err != nil { - return nil, err - } - raw.SetPayload(all) - } - raw.SetPayloadSize(uint64(len(raw.Payload()))) - - addr := newAddress(raw.ContainerID(), raw.ID()) - t.Objects[addr.String()] = raw.Object() - return raw.ID(), nil -} - -func (t *TestPool) DeleteObject(ctx context.Context, params *client.DeleteObjectParams, option ...pool.CallOption) error { - delete(t.Objects, params.Address().String()) - return nil -} - -func (t *TestPool) GetObject(ctx context.Context, params *client.GetObjectParams, option ...pool.CallOption) (*object.Object, error) { - if obj, ok := t.Objects[params.Address().String()]; ok { - if params.PayloadWriter() != nil { - _, err := params.PayloadWriter().Write(obj.Payload()) - if err != nil { - return nil, err - } - } - return obj, nil - } - - return nil, fmt.Errorf("object not found " + params.Address().String()) -} - -func (t *TestPool) GetObjectHeader(ctx context.Context, params *client.ObjectHeaderParams, option ...pool.CallOption) (*object.Object, error) { - p := new(client.GetObjectParams).WithAddress(params.Address()) - return t.GetObject(ctx, p) -} - -func (t *TestPool) ObjectPayloadRangeData(ctx context.Context, params *client.RangeDataParams, option ...pool.CallOption) ([]byte, error) { - panic("implement me") -} - -func (t *TestPool) ObjectPayloadRangeSHA256(ctx context.Context, params *client.RangeChecksumParams, option ...pool.CallOption) ([][32]byte, error) { - panic("implement me") -} - -func (t *TestPool) ObjectPayloadRangeTZ(ctx context.Context, params *client.RangeChecksumParams, option ...pool.CallOption) ([][64]byte, error) { - panic("implement me") -} - -func (t *TestPool) SearchObject(ctx context.Context, params *client.SearchObjectParams, option ...pool.CallOption) ([]*object.ID, error) { - cidStr := params.ContainerID().String() - - var res []*object.ID - - if len(params.SearchFilters()) == 1 { - for k, v := range t.Objects { - if strings.Contains(k, cidStr) { - res = append(res, v.ID()) - } - } - return res, nil - } - - filter := params.SearchFilters()[1] - if len(params.SearchFilters()) != 2 || filter.Operation() != object.MatchStringEqual || - (filter.Header() != object.AttributeFileName && filter.Header() != "S3-System-name") { - return nil, fmt.Errorf("usupported filters") - } - - for k, v := range t.Objects { - if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) { - res = append(res, v.ID()) - } - } - - return res, nil -} - -func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool { - for _, attr := range attributes { - if attr.Key() == filter.Header() && attr.Value() == filter.Value() { - return true - } - } - - return false -} - -func (t *TestPool) PutContainer(ctx context.Context, container *container.Container, option ...pool.CallOption) (*cid.ID, error) { - b := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, b); err != nil { - return nil, err - } - - id := cid.New() - id.SetSHA256(sha256.Sum256(b)) - t.Containers[id.String()] = container - - return id, nil -} - -func (t *TestPool) GetContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*container.Container, error) { - for k, v := range t.Containers { - if k == id.String() { - return v, nil - } - } - - return nil, fmt.Errorf("container not found " + id.String()) -} - -func (t *TestPool) ListContainers(ctx context.Context, id *owner.ID, option ...pool.CallOption) ([]*cid.ID, error) { - var res []*cid.ID - for k := range t.Containers { - cID := cid.New() - if err := cID.Parse(k); err != nil { - return nil, err - } - res = append(res, cID) - } - - return res, nil -} - -func (t *TestPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...pool.CallOption) error { - delete(t.Containers, id.String()) - return nil -} - -func (t *TestPool) GetEACL(ctx context.Context, id *cid.ID, option ...pool.CallOption) (*eacl.Table, error) { - panic("implement me") -} - -func (t *TestPool) Balance(ctx context.Context, owner *owner.ID, opts ...pool.CallOption) (*accounting.Decimal, error) { - panic("implement me") -} - -func (t *TestPool) SetEACL(ctx context.Context, table *eacl.Table, option ...pool.CallOption) error { - panic("implement me") -} - -func (t *TestPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...pool.CallOption) error { - panic("implement me") -} - -func (t *TestPool) Connection() (pool.Client, *session.Token, error) { - panic("implement me") -} - -func (t *TestPool) Close() { - panic("implement me") -} - -func (t *TestPool) OwnerID() *owner.ID { - return nil -} - -func (t *TestPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error { - return nil -} - -func newAddress(cid *cid.ID, oid *object.ID) *object.Address { - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(oid) - return address -} diff --git a/cmd/s3-gw/neofs.go b/cmd/s3-gw/neofs.go index ea213fd..3211d28 100644 --- a/cmd/s3-gw/neofs.go +++ b/cmd/s3-gw/neofs.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/nspcc-dev/neofs-s3-gw/api/layer" + layer "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-s3-gw/internal/neofs" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -17,13 +17,13 @@ type layerNeoFS struct { func (x *layerNeoFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreate) (*cid.ID, error) { return x.NeoFS.CreateContainer(ctx, neofs.PrmContainerCreate{ - Creator: prm.Creator, - Policy: prm.Policy, - Name: prm.Name, - Time: prm.Time, - BasicACL: prm.BasicACL, - SessionToken: prm.SessionToken, - LocationConstraintAttribute: prm.LocationConstraintAttribute, + Creator: prm.Creator, + Policy: prm.Policy, + Name: prm.Name, + Time: prm.Time, + BasicACL: prm.BasicACL, + SessionToken: prm.SessionToken, + AdditionalAttributes: prm.AdditionalAttributes, }) } diff --git a/internal/neofs/neofs.go b/internal/neofs/neofs.go index 561d780..3165463 100644 --- a/internal/neofs/neofs.go +++ b/internal/neofs/neofs.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/nspcc-dev/neofs-s3-gw/api/layer" + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" "github.com/nspcc-dev/neofs-s3-gw/authmate" "github.com/nspcc-dev/neofs-s3-gw/creds/tokens" "github.com/nspcc-dev/neofs-sdk-go/acl" @@ -123,8 +123,8 @@ type PrmContainerCreate struct { // Token of the container's creation session (optional, nil means session absence). SessionToken *session.Token - // Attribute for LocationConstraint parameter (optional). - LocationConstraintAttribute *container.Attribute + // Attributes for optional parameters. + AdditionalAttributes [][2]string } // CreateContainer constructs new container from the parameters and saves it in NeoFS @@ -143,11 +143,8 @@ func (x *NeoFS) CreateContainer(ctx context.Context, prm PrmContainerCreate) (*c cnrOptions = append(cnrOptions, container.WithAttribute(container.AttributeName, prm.Name)) } - if prm.LocationConstraintAttribute != nil { - cnrOptions = append(cnrOptions, container.WithAttribute( - prm.LocationConstraintAttribute.Key(), - prm.LocationConstraintAttribute.Value(), - )) + for _, attr := range prm.AdditionalAttributes { + cnrOptions = append(cnrOptions, container.WithAttribute(attr[0], attr[1])) } cnr := container.New(cnrOptions...) @@ -319,9 +316,9 @@ func (x *NeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (*oid.ID, // using connection pool. // // Returns any error encountered which prevented the selection to be finished. -// Returns layer.ErrAccessDenied on access violation. -func (x *NeoFS) SelectObjects(ctx context.Context, prm layer.PrmObjectSelect) ([]oid.ID, error) { - var filters object.SearchFilters +// Returns neofs.ErrAccessDenied on access violation. +func (x *NeoFS) SelectObjects(ctx context.Context, prm neofs.PrmObjectSelect) ([]oid.ID, error) { + filters := object.NewSearchFilters() filters.AddRootFilter() if prm.ExactAttribute[0] != "" { @@ -356,7 +353,7 @@ func (x *NeoFS) SelectObjects(ctx context.Context, prm layer.PrmObjectSelect) ([ if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return nil, layer.ErrAccessDenied + return nil, neofs.ErrAccessDenied } return nil, fmt.Errorf("read object list: %w", err) @@ -366,7 +363,7 @@ func (x *NeoFS) SelectObjects(ctx context.Context, prm layer.PrmObjectSelect) ([ } // wraps io.ReadCloser and transforms Read errors related to access violation -// to layer.ErrAccessDenied. +// to neofs.ErrAccessDenied. type payloadReader struct { io.ReadCloser } @@ -376,7 +373,7 @@ func (x payloadReader) Read(p []byte) (int, error) { if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return n, layer.ErrAccessDenied + return n, neofs.ErrAccessDenied } } @@ -389,8 +386,8 @@ func (x payloadReader) Read(p []byte) (int, error) { // * else GetObject is called. // // Returns any error encountered which prevented the object to be read. -// Returns layer.ErrAccessDenied on access violation. -func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer.ObjectPart, error) { +// Returns neofs.ErrAccessDenied on access violation. +func (x *NeoFS) ReadObject(ctx context.Context, prm neofs.PrmObjectRead) (*neofs.ObjectPart, error) { var addr address.Address addr.SetContainerID(&prm.Container) addr.SetObjectID(&prm.Object) @@ -409,7 +406,7 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return nil, layer.ErrAccessDenied + return nil, neofs.ErrAccessDenied } return nil, fmt.Errorf("init full object reading via connection pool: %w", err) @@ -424,7 +421,7 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer res.Header.SetPayload(payload) - return &layer.ObjectPart{ + return &neofs.ObjectPart{ Head: &res.Header, }, nil } @@ -433,13 +430,13 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return nil, layer.ErrAccessDenied + return nil, neofs.ErrAccessDenied } return nil, fmt.Errorf("read object header via connection pool: %w", err) } - return &layer.ObjectPart{ + return &neofs.ObjectPart{ Head: hdr, }, nil } else if prm.PayloadRange[0]+prm.PayloadRange[1] == 0 { @@ -447,13 +444,13 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return nil, layer.ErrAccessDenied + return nil, neofs.ErrAccessDenied } return nil, fmt.Errorf("init full payload range reading via connection pool: %w", err) } - return &layer.ObjectPart{ + return &neofs.ObjectPart{ Payload: res.Payload, }, nil } @@ -462,13 +459,13 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return nil, layer.ErrAccessDenied + return nil, neofs.ErrAccessDenied } return nil, fmt.Errorf("init payload range reading via connection pool: %w", err) } - return &layer.ObjectPart{ + return &neofs.ObjectPart{ Payload: payloadReader{res}, }, nil } @@ -479,7 +476,7 @@ func (x *NeoFS) ReadObject(ctx context.Context, prm layer.PrmObjectRead) (*layer // Returns ErrAccessDenied on remove access violation. // // Returns any error encountered which prevented the removal request to be sent. -func (x *NeoFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) error { +func (x *NeoFS) DeleteObject(ctx context.Context, prm neofs.PrmObjectDelete) error { var addr address.Address addr.SetContainerID(&prm.Container) addr.SetObjectID(&prm.Object) @@ -496,7 +493,7 @@ func (x *NeoFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) err if err != nil { // TODO: (neofs-s3-gw#367) use NeoFS SDK API to check the status return if strings.Contains(err.Error(), "access to operation") && strings.Contains(err.Error(), "is denied by") { - return layer.ErrAccessDenied + return neofs.ErrAccessDenied } return fmt.Errorf("mark object removal via connection pool: %w", err) @@ -521,7 +518,7 @@ func (x *AuthmateNeoFS) CreateContainer(ctx context.Context, prm authmate.PrmCon } func (x *AuthmateNeoFS) ReadObjectPayload(ctx context.Context, addr address.Address) ([]byte, error) { - res, err := x.NeoFS.ReadObject(ctx, layer.PrmObjectRead{ + res, err := x.NeoFS.ReadObject(ctx, neofs.PrmObjectRead{ Container: *addr.ContainerID(), Object: *addr.ObjectID(), WithPayload: true, diff --git a/internal/neofstest/neofs_mock.go b/internal/neofstest/neofs_mock.go new file mode 100644 index 0000000..a47c09b --- /dev/null +++ b/internal/neofstest/neofs_mock.go @@ -0,0 +1,251 @@ +package neofstest + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "io" + "strconv" + "strings" + + "github.com/nspcc-dev/neofs-s3-gw/api/layer/neofs" + "github.com/nspcc-dev/neofs-sdk-go/container" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "github.com/nspcc-dev/neofs-sdk-go/object/id/test" + "github.com/nspcc-dev/neofs-sdk-go/owner" +) + +const objectSystemAttributeName = "S3-System-name" + +type TestNeoFS struct { + neofs.NeoFS + + objects map[string]*object.Object + containers map[string]*container.Container + currentEpoch uint64 +} + +func NewTestNeoFS() *TestNeoFS { + return &TestNeoFS{ + objects: make(map[string]*object.Object), + containers: make(map[string]*container.Container), + } +} + +func (t *TestNeoFS) CurrentEpoch() uint64 { + return t.currentEpoch +} + +func (t *TestNeoFS) Objects() []*object.Object { + res := make([]*object.Object, 0, len(t.objects)) + + for _, obj := range t.objects { + res = append(res, obj) + } + + return res +} + +func (t *TestNeoFS) AddObject(key string, obj *object.Object) { + t.objects[key] = obj +} + +func (t *TestNeoFS) ContainerID(name string) (*cid.ID, error) { + for id, cnr := range t.containers { + for _, attr := range cnr.Attributes() { + if attr.Key() == container.AttributeName && attr.Value() == name { + cnrID := cid.New() + return cnrID, cnrID.Parse(id) + } + } + } + return nil, fmt.Errorf("not found") +} + +func (t *TestNeoFS) CreateContainer(_ context.Context, prm neofs.PrmContainerCreate) (*cid.ID, error) { + opts := []container.Option{ + container.WithOwnerID(&prm.Creator), + container.WithPolicy(&prm.Policy), + container.WithCustomBasicACL(prm.BasicACL), + container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(prm.Time.Unix(), 10)), + } + + if prm.Name != "" { + opts = append(opts, container.WithAttribute(container.AttributeName, prm.Name)) + } + + for _, attr := range prm.AdditionalAttributes { + opts = append(opts, container.WithAttribute(attr[0], attr[1])) + } + + cnr := container.New(opts...) + cnr.SetSessionToken(prm.SessionToken) + + if prm.Name != "" { + container.SetNativeName(cnr, prm.Name) + } + + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return nil, err + } + + id := cid.New() + id.SetSHA256(sha256.Sum256(b)) + t.containers[id.String()] = cnr + + return id, nil +} + +func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) { + for k, v := range t.containers { + if k == id.String() { + return v, nil + } + } + + return nil, fmt.Errorf("container not found " + id.String()) +} + +func (t *TestNeoFS) UserContainers(_ context.Context, _ owner.ID) ([]cid.ID, error) { + var res []cid.ID + for k := range t.containers { + var idCnr cid.ID + if err := idCnr.Parse(k); err != nil { + return nil, err + } + res = append(res, idCnr) + } + + return res, nil +} + +func (t *TestNeoFS) SelectObjects(_ context.Context, prm neofs.PrmObjectSelect) ([]oid.ID, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + + if prm.FilePrefix != "" { + filters.AddFilter(object.AttributeFileName, prm.FilePrefix, object.MatchCommonPrefix) + } + + if prm.ExactAttribute[0] != "" { + filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual) + } + + cidStr := prm.Container.String() + + var res []oid.ID + + if len(filters) == 1 { + for k, v := range t.objects { + if strings.Contains(k, cidStr) { + res = append(res, *v.ID()) + } + } + return res, nil + } + + filter := filters[1] + if len(filters) != 2 || filter.Operation() != object.MatchStringEqual || + (filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) { + return nil, fmt.Errorf("usupported filters") + } + + for k, v := range t.objects { + if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) { + res = append(res, *v.ID()) + } + } + + return res, nil +} + +func (t *TestNeoFS) ReadObject(_ context.Context, prm neofs.PrmObjectRead) (*neofs.ObjectPart, error) { + var addr address.Address + addr.SetContainerID(&prm.Container) + addr.SetObjectID(&prm.Object) + + sAddr := addr.String() + + if obj, ok := t.objects[sAddr]; ok { + return &neofs.ObjectPart{ + Head: obj, + Payload: io.NopCloser(bytes.NewReader(obj.Payload())), + }, nil + } + + return nil, fmt.Errorf("object not found " + addr.String()) +} + +func (t *TestNeoFS) CreateObject(_ context.Context, prm neofs.PrmObjectCreate) (*oid.ID, error) { + id := test.ID() + + attrs := make([]*object.Attribute, 0) + + if prm.Filename != "" { + a := object.NewAttribute() + a.SetKey(object.AttributeFileName) + a.SetValue(prm.Filename) + attrs = append(attrs, a) + } + + for i := range prm.Attributes { + a := object.NewAttribute() + a.SetKey(prm.Attributes[i][0]) + a.SetValue(prm.Attributes[i][1]) + attrs = append(attrs, a) + } + + obj := object.New() + obj.SetContainerID(&prm.Container) + obj.SetID(id) + obj.SetPayloadSize(prm.PayloadSize) + obj.SetAttributes(attrs...) + obj.SetCreationEpoch(t.currentEpoch) + t.currentEpoch++ + + if prm.Payload != nil { + all, err := io.ReadAll(prm.Payload) + if err != nil { + return nil, err + } + obj.SetPayload(all) + obj.SetPayloadSize(uint64(len(all))) + } + + addr := newAddress(obj.ContainerID(), obj.ID()) + t.objects[addr.String()] = obj + return obj.ID(), nil +} + +func (t *TestNeoFS) DeleteObject(_ context.Context, prm neofs.PrmObjectDelete) error { + var addr address.Address + addr.SetContainerID(&prm.Container) + addr.SetObjectID(&prm.Object) + + delete(t.objects, addr.String()) + + return nil +} + +func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool { + for _, attr := range attributes { + if attr.Key() == filter.Header() && attr.Value() == filter.Value() { + return true + } + } + + return false +} + +func newAddress(cid *cid.ID, oid *oid.ID) *address.Address { + addr := address.NewAddress() + addr.SetContainerID(cid) + addr.SetObjectID(oid) + return addr +}