TrueCloudLab/frostfs-rest-gw
Archived
15
0
Fork 6
Code Issues 2 Pull requests 1 Projects Releases Activity Actions

[#1] Support GET/RANGE object payload

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-04-20 17:10:43 +03:00 committed by Alex Vanin
parent dc1926f9c6
commit 3727f5561d
8 changed files with 395 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
cmd/neofs-rest-gw/integration_test.go
View file

@ -273,12 +273,13 @@ func restObjectPut(ctx context.Context, t *testing.T, clientPool *pool.Pool, cnr
}
func restObjectGet(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID) {
content := []byte("some content")
attributes := map[string]string{
object.AttributeFileName: "get-obj-name",
"user-attribute": "user value",
}
objID := createObject(ctx, t, p, cnrID, attributes, []byte("some content"))
objID := createObject(ctx, t, p, cnrID, attributes, content)
bearer := &models.Bearer{
Object: []*models.Record{{
@ -309,10 +310,48 @@ func restObjectGet(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.I
require.Equal(t, objID.String(), *objInfo.ObjectID)
require.Equal(t, p.OwnerID().String(), *objInfo.OwnerID)
require.Equal(t, len(attributes), len(objInfo.Attributes))
require.Equal(t, int64(len(content)), *objInfo.ObjectSize)
contentData, err := base64.StdEncoding.DecodeString(objInfo.Payload)
require.NoError(t, err)
require.Equal(t, content, contentData)
for _, attr := range objInfo.Attributes {
require.Equal(t, attributes[*attr.Key], *attr.Value)
}
// check max-payload-size params
query = make(url.Values)
query.Add(walletConnectQuery, strconv.FormatBool(useWalletConnect))
query.Add("max-payload-size", "0")
request, err = http.NewRequest(http.MethodGet, testHost+"/v1/objects/"+cnrID.String()+"/"+objID.String()+"?"+query.Encode(), nil)
require.NoError(t, err)
prepareCommonHeaders(request.Header, bearerToken)
objInfo = &models.ObjectInfo{}
doRequest(t, httpClient, request, http.StatusOK, objInfo)
require.Empty(t, objInfo.Payload)
require.Equal(t, int64(0), *objInfo.PayloadSize)
// check range params
rangeLength := 4
query = make(url.Values)
query.Add(walletConnectQuery, strconv.FormatBool(useWalletConnect))
query.Add("range-offset", "0")
query.Add("range-length", strconv.Itoa(rangeLength))
request, err = http.NewRequest(http.MethodGet, testHost+"/v1/objects/"+cnrID.String()+"/"+objID.String()+"?"+query.Encode(), nil)
require.NoError(t, err)
prepareCommonHeaders(request.Header, bearerToken)
objInfo = &models.ObjectInfo{}
doRequest(t, httpClient, request, http.StatusOK, objInfo)
require.Equal(t, int64(rangeLength), *objInfo.PayloadSize)
contentData, err = base64.StdEncoding.DecodeString(objInfo.Payload)
require.NoError(t, err)
require.Equal(t, content[:rangeLength], contentData)
}
func restObjectDelete(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID) {

37
gen/models/object_info.go
View file

@ -33,9 +33,20 @@ type ObjectInfo struct {
// Required: true
ObjectID *string `json:"objectId"`
// Object full payload size
// Required: true
ObjectSize *int64 `json:"objectSize"`
// owner Id
// Required: true
OwnerID *string `json:"ownerId"`
// Base64 encoded object payload
Payload string `json:"payload,omitempty"`
// Payload size in response
// Required: true
PayloadSize *int64 `json:"payloadSize"`
}
// Validate validates this object info
@ -54,10 +65,18 @@ func (m *ObjectInfo) Validate(formats strfmt.Registry) error {
res = append(res, err)
}
if err := m.validateObjectSize(formats); err != nil {
res = append(res, err)
}
if err := m.validateOwnerID(formats); err != nil {
res = append(res, err)
}
if err := m.validatePayloadSize(formats); err != nil {
res = append(res, err)
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
@ -109,6 +128,15 @@ func (m *ObjectInfo) validateObjectID(formats strfmt.Registry) error {
return nil
}
func (m *ObjectInfo) validateObjectSize(formats strfmt.Registry) error {
if err := validate.Required("objectSize", "body", m.ObjectSize); err != nil {
return err
}
return nil
}
func (m *ObjectInfo) validateOwnerID(formats strfmt.Registry) error {
if err := validate.Required("ownerId", "body", m.OwnerID); err != nil {
@ -118,6 +146,15 @@ func (m *ObjectInfo) validateOwnerID(formats strfmt.Registry) error {
return nil
}
func (m *ObjectInfo) validatePayloadSize(formats strfmt.Registry) error {
if err := validate.Required("payloadSize", "body", m.PayloadSize); err != nil {
return err
}
return nil
}
// ContextValidate validate this object info based on the context it is used
func (m *ObjectInfo) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
var res []error

80
gen/restapi/embedded_spec.go
View file

@ -454,8 +454,29 @@ func init() {
},
"/objects/{containerId}/{objectId}": {
"get": {
"summary": "Get object info by address and",
"summary": "Get object info by address",
"operationId": "getObjectInfo",
"parameters": [
{
"type": "integer",
"name": "range-offset",
"in": "query"
},
{
"minimum": 1,
"type": "integer",
"name": "range-length",
"in": "query"
},
{
"maximum": 524288000,
"type": "integer",
"default": 4194304,
"description": "Max payload size (in bytes) that can be included in the response.\nIf the actual size is greater than this params the payload won't be included in the response.\n",
"name": "max-payload-size",
"in": "query"
}
],
"responses": {
"200": {
"description": "Object info",
@ -729,7 +750,9 @@ func init() {
"containerId",
"objectId",
"ownerId",
"attributes"
"attributes",
"objectSize",
"payloadSize"
],
"properties": {
"attributes": {
@ -744,8 +767,20 @@ func init() {
"objectId": {
"type": "string"
},
"objectSize": {
"description": "Object full payload size",
"type": "integer"
},
"ownerId": {
"type": "string"
},
"payload": {
"description": "Base64 encoded object payload",
"type": "string"
},
"payloadSize": {
"description": "Payload size in response",
"type": "integer"
}
},
"example": {
@ -1532,8 +1567,31 @@ func init() {
},
"/objects/{containerId}/{objectId}": {
"get": {
"summary": "Get object info by address and",
"summary": "Get object info by address",
"operationId": "getObjectInfo",
"parameters": [
{
"minimum": 0,
"type": "integer",
"name": "range-offset",
"in": "query"
},
{
"minimum": 1,
"type": "integer",
"name": "range-length",
"in": "query"
},
{
"maximum": 524288000,
"minimum": 0,
"type": "integer",
"default": 4194304,
"description": "Max payload size (in bytes) that can be included in the response.\nIf the actual size is greater than this params the payload won't be included in the response.\n",
"name": "max-payload-size",
"in": "query"
}
],
"responses": {
"200": {
"description": "Object info",
@ -1827,7 +1885,9 @@ func init() {
"containerId",
"objectId",
"ownerId",
"attributes"
"attributes",
"objectSize",
"payloadSize"
],
"properties": {
"attributes": {
@ -1842,8 +1902,20 @@ func init() {
"objectId": {
"type": "string"
},
"objectSize": {
"description": "Object full payload size",
"type": "integer"
},
"ownerId": {
"type": "string"
},
"payload": {
"description": "Base64 encoded object payload",
"type": "string"
},
"payloadSize": {
"description": "Payload size in response",
"type": "integer"
}
},
"example": {

2
gen/restapi/operations/get_object_info.go
View file

@ -33,7 +33,7 @@ func NewGetObjectInfo(ctx *middleware.Context, handler GetObjectInfoHandler) *Ge
/* GetObjectInfo swagger:route GET /objects/{containerId}/{objectId} getObjectInfo
Get object info by address and
Get object info by address
*/
type GetObjectInfo struct {

154
gen/restapi/operations/get_object_info_parameters.go
View file

@ -23,10 +23,14 @@ func NewGetObjectInfoParams() GetObjectInfoParams {
var (
// initialize parameters with default values
maxPayloadSizeDefault = int64(4.194304e+06)
walletConnectDefault = bool(false)
)
return GetObjectInfoParams{
MaxPayloadSize: &maxPayloadSizeDefault,
WalletConnect: &walletConnectDefault,
}
}
@ -55,11 +59,30 @@ type GetObjectInfoParams struct {
In: path
*/
ContainerID string
/*Max payload size (in bytes) that can be included in the response.
If the actual size is greater than this params the payload won't be included in the response.
Maximum: 5.24288e+08
Minimum: 0
In: query
Default: 4.194304e+06
*/
MaxPayloadSize *int64
/*Base58 encoded object id
Required: true
In: path
*/
ObjectID string
/*
Minimum: 1
In: query
*/
RangeLength *int64
/*
Minimum: 0
In: query
*/
RangeOffset *int64
/*Use wallect connect signature scheme or not
In: query
Default: false
@ -91,11 +114,26 @@ func (o *GetObjectInfoParams) BindRequest(r *http.Request, route *middleware.Mat
res = append(res, err)
}
qMaxPayloadSize, qhkMaxPayloadSize, _ := qs.GetOK("max-payload-size")
if err := o.bindMaxPayloadSize(qMaxPayloadSize, qhkMaxPayloadSize, route.Formats); err != nil {
res = append(res, err)
}
rObjectID, rhkObjectID, _ := route.Params.GetOK("objectId")
if err := o.bindObjectID(rObjectID, rhkObjectID, route.Formats); err != nil {
res = append(res, err)
}
qRangeLength, qhkRangeLength, _ := qs.GetOK("range-length")
if err := o.bindRangeLength(qRangeLength, qhkRangeLength, route.Formats); err != nil {
res = append(res, err)
}
qRangeOffset, qhkRangeOffset, _ := qs.GetOK("range-offset")
if err := o.bindRangeOffset(qRangeOffset, qhkRangeOffset, route.Formats); err != nil {
res = append(res, err)
}
qWalletConnect, qhkWalletConnect, _ := qs.GetOK("walletConnect")
if err := o.bindWalletConnect(qWalletConnect, qhkWalletConnect, route.Formats); err != nil {
res = append(res, err)
@ -160,6 +198,48 @@ func (o *GetObjectInfoParams) bindContainerID(rawData []string, hasKey bool, for
return nil
}
// bindMaxPayloadSize binds and validates parameter MaxPayloadSize from query.
func (o *GetObjectInfoParams) bindMaxPayloadSize(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
// Default values have been previously initialized by NewGetObjectInfoParams()
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("max-payload-size", "query", "int64", raw)
}
o.MaxPayloadSize = &value
if err := o.validateMaxPayloadSize(formats); err != nil {
return err
}
return nil
}
// validateMaxPayloadSize carries on validations for parameter MaxPayloadSize
func (o *GetObjectInfoParams) validateMaxPayloadSize(formats strfmt.Registry) error {
if err := validate.MinimumInt("max-payload-size", "query", *o.MaxPayloadSize, 0, false); err != nil {
return err
}
if err := validate.MaximumInt("max-payload-size", "query", *o.MaxPayloadSize, 5.24288e+08, false); err != nil {
return err
}
return nil
}
// bindObjectID binds and validates parameter ObjectID from path.
func (o *GetObjectInfoParams) bindObjectID(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
@ -174,6 +254,80 @@ func (o *GetObjectInfoParams) bindObjectID(rawData []string, hasKey bool, format
return nil
}
// bindRangeLength binds and validates parameter RangeLength from query.
func (o *GetObjectInfoParams) bindRangeLength(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("range-length", "query", "int64", raw)
}
o.RangeLength = &value
if err := o.validateRangeLength(formats); err != nil {
return err
}
return nil
}
// validateRangeLength carries on validations for parameter RangeLength
func (o *GetObjectInfoParams) validateRangeLength(formats strfmt.Registry) error {
if err := validate.MinimumInt("range-length", "query", *o.RangeLength, 1, false); err != nil {
return err
}
return nil
}
// bindRangeOffset binds and validates parameter RangeOffset from query.
func (o *GetObjectInfoParams) bindRangeOffset(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("range-offset", "query", "int64", raw)
}
o.RangeOffset = &value
if err := o.validateRangeOffset(formats); err != nil {
return err
}
return nil
}
// validateRangeOffset carries on validations for parameter RangeOffset
func (o *GetObjectInfoParams) validateRangeOffset(formats strfmt.Registry) error {
if err := validate.MinimumInt("range-offset", "query", *o.RangeOffset, 0, false); err != nil {
return err
}
return nil
}
// bindWalletConnect binds and validates parameter WalletConnect from query.
func (o *GetObjectInfoParams) bindWalletConnect(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string

2
handlers/auth_test.go
View file

@ -61,6 +61,6 @@ func TestSign(t *testing.T) {
Key: pubKeyHex,
}
_, err = prepareBearerToken(bt)
_, err = prepareBearerToken(bt, false)
require.NoError(t, err)
}

56
handlers/objects.go
View file

@ -5,7 +5,6 @@ import (
"crypto/ecdsa"
"encoding/base64"
"fmt"
"github.com/go-openapi/runtime/middleware"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-api-go/v2/acl"
@ -20,6 +19,8 @@ import (
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/token"
"go.uber.org/zap"
"io"
"strings"
)
// PutObjects handler that uploads object to NeoFS.
@ -88,6 +89,7 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo
btoken, err := getBearerToken(principal, params.XBearerSignature, params.XBearerSignatureKey, *params.WalletConnect)
if err != nil {
a.log.Error("get bearer token", zap.Error(err))
return errorResponse.WithPayload(NewError(err))
}
@ -97,6 +99,7 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo
objInfo, err := a.pool.HeadObject(ctx, prm)
if err != nil {
a.log.Error("head object", zap.Error(err))
return errorResponse.WithPayload(NewError(err))
}
@ -105,6 +108,8 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo
resp.ObjectID = NewString(params.ObjectID)
resp.OwnerID = NewString(objInfo.OwnerID().String())
resp.Attributes = make([]*models.Attribute, len(objInfo.Attributes()))
resp.ObjectSize = NewInteger(int64(objInfo.PayloadSize()))
resp.PayloadSize = NewInteger(0)
for i, attr := range objInfo.Attributes() {
resp.Attributes[i] = &models.Attribute{
@ -113,6 +118,55 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo
}
}
var prmRange pool.PrmObjectRange
prmRange.SetAddress(*addr)
prmRange.UseBearer(btoken)
var offset, length uint64
if params.RangeOffset != nil || params.RangeLength != nil {
if params.RangeOffset == nil || params.RangeLength == nil {
a.log.Error("both offset and length must be provided")
return errorResponse.WithPayload(NewError(fmt.Errorf("both offset and length must be provided")))
}
offset = uint64(*params.RangeOffset)
length = uint64(*params.RangeLength)
} else {
length = objInfo.PayloadSize()
}
prmRange.SetOffset(offset)
prmRange.SetLength(length)
if uint64(*params.MaxPayloadSize) < length {
return operations.NewGetObjectInfoOK().WithPayload(&resp)
}
rangeRes, err := a.pool.ObjectRange(ctx, prmRange)
if err != nil {
a.log.Error("range object", zap.Error(err))
return errorResponse.WithPayload(NewError(err))
}
defer func() {
if err = rangeRes.Close(); err != nil {
a.log.Error("close range result", zap.Error(err))
}
}()
sb := new(strings.Builder)
encoder := base64.NewEncoder(base64.StdEncoding, sb)
payloadSize, err := io.Copy(encoder, rangeRes)
if err != nil {
a.log.Error("encode object payload", zap.Error(err))
return errorResponse.WithPayload(NewError(err))
}
if err = encoder.Close(); err != nil {
a.log.Error("close encoder", zap.Error(err))
return errorResponse.WithPayload(NewError(err))
}
resp.Payload = sb.String()
resp.PayloadSize = NewInteger(payloadSize)
return operations.NewGetObjectInfoOK().WithPayload(&resp)
}

33
spec/rest.yaml
View file

@ -184,7 +184,25 @@ paths:
- $ref: '#/parameters/objectId'
get:
operationId: getObjectInfo
summary: Get object info by address and
summary: Get object info by address
parameters:
- in: query
name: range-offset
type: integer
minimum: 0
- in: query
name: range-length
type: integer
minimum: 1
- in: query
name: max-payload-size
type: integer
default: 4194304
minimum: 0
maximum: 524288000
description: |
Max payload size (in bytes) that can be included in the response.
If the actual size is greater than this params the payload won't be included in the response.
responses:
200:
description: Object info
@ -613,11 +631,22 @@ definitions:
type: array
items:
$ref: '#/definitions/Attribute'
objectSize:
type: integer
description: Object full payload size
payloadSize:
type: integer
description: Payload size in response
payload:
type: string
description: Base64 encoded object payload
required:
- containerId
- objectId
- ownerId
- attributes
- objectSize
- payloadSize
example:
containerId: 5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv
objectId: 8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd
@ -633,7 +662,7 @@ definitions:
containerId:
type: string
objectId:
type: string
type: string
required:
- containerId
- objectId