From 3727f5561d8cc557fd1c9aa61718fae92cb95db0 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 20 Apr 2022 17:10:43 +0300 Subject: [PATCH] [#1] Support GET/RANGE object payload Signed-off-by: Denis Kirillov --- cmd/neofs-rest-gw/integration_test.go | 41 ++++- gen/models/object_info.go | 37 +++++ gen/restapi/embedded_spec.go | 80 ++++++++- gen/restapi/operations/get_object_info.go | 2 +- .../operations/get_object_info_parameters.go | 154 ++++++++++++++++++ handlers/auth_test.go | 2 +- handlers/objects.go | 56 ++++++- spec/rest.yaml | 33 +++- 8 files changed, 395 insertions(+), 10 deletions(-) diff --git a/cmd/neofs-rest-gw/integration_test.go b/cmd/neofs-rest-gw/integration_test.go index 5dea114..230c5e3 100644 --- a/cmd/neofs-rest-gw/integration_test.go +++ b/cmd/neofs-rest-gw/integration_test.go @@ -273,12 +273,13 @@ func restObjectPut(ctx context.Context, t *testing.T, clientPool *pool.Pool, cnr } func restObjectGet(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID) { + content := []byte("some content") attributes := map[string]string{ object.AttributeFileName: "get-obj-name", "user-attribute": "user value", } - objID := createObject(ctx, t, p, cnrID, attributes, []byte("some content")) + objID := createObject(ctx, t, p, cnrID, attributes, content) bearer := &models.Bearer{ Object: []*models.Record{{ @@ -309,10 +310,48 @@ func restObjectGet(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.I require.Equal(t, objID.String(), *objInfo.ObjectID) require.Equal(t, p.OwnerID().String(), *objInfo.OwnerID) require.Equal(t, len(attributes), len(objInfo.Attributes)) + require.Equal(t, int64(len(content)), *objInfo.ObjectSize) + + contentData, err := base64.StdEncoding.DecodeString(objInfo.Payload) + require.NoError(t, err) + require.Equal(t, content, contentData) for _, attr := range objInfo.Attributes { require.Equal(t, attributes[*attr.Key], *attr.Value) } + + // check max-payload-size params + query = make(url.Values) + query.Add(walletConnectQuery, strconv.FormatBool(useWalletConnect)) + query.Add("max-payload-size", "0") + + request, err = http.NewRequest(http.MethodGet, testHost+"/v1/objects/"+cnrID.String()+"/"+objID.String()+"?"+query.Encode(), nil) + require.NoError(t, err) + prepareCommonHeaders(request.Header, bearerToken) + + objInfo = &models.ObjectInfo{} + doRequest(t, httpClient, request, http.StatusOK, objInfo) + require.Empty(t, objInfo.Payload) + require.Equal(t, int64(0), *objInfo.PayloadSize) + + // check range params + rangeLength := 4 + query = make(url.Values) + query.Add(walletConnectQuery, strconv.FormatBool(useWalletConnect)) + query.Add("range-offset", "0") + query.Add("range-length", strconv.Itoa(rangeLength)) + + request, err = http.NewRequest(http.MethodGet, testHost+"/v1/objects/"+cnrID.String()+"/"+objID.String()+"?"+query.Encode(), nil) + require.NoError(t, err) + prepareCommonHeaders(request.Header, bearerToken) + + objInfo = &models.ObjectInfo{} + doRequest(t, httpClient, request, http.StatusOK, objInfo) + require.Equal(t, int64(rangeLength), *objInfo.PayloadSize) + + contentData, err = base64.StdEncoding.DecodeString(objInfo.Payload) + require.NoError(t, err) + require.Equal(t, content[:rangeLength], contentData) } func restObjectDelete(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID) { diff --git a/gen/models/object_info.go b/gen/models/object_info.go index db5f14b..090c062 100644 --- a/gen/models/object_info.go +++ b/gen/models/object_info.go @@ -33,9 +33,20 @@ type ObjectInfo struct { // Required: true ObjectID *string `json:"objectId"` + // Object full payload size + // Required: true + ObjectSize *int64 `json:"objectSize"` + // owner Id // Required: true OwnerID *string `json:"ownerId"` + + // Base64 encoded object payload + Payload string `json:"payload,omitempty"` + + // Payload size in response + // Required: true + PayloadSize *int64 `json:"payloadSize"` } // Validate validates this object info @@ -54,10 +65,18 @@ func (m *ObjectInfo) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateObjectSize(formats); err != nil { + res = append(res, err) + } + if err := m.validateOwnerID(formats); err != nil { res = append(res, err) } + if err := m.validatePayloadSize(formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -109,6 +128,15 @@ func (m *ObjectInfo) validateObjectID(formats strfmt.Registry) error { return nil } +func (m *ObjectInfo) validateObjectSize(formats strfmt.Registry) error { + + if err := validate.Required("objectSize", "body", m.ObjectSize); err != nil { + return err + } + + return nil +} + func (m *ObjectInfo) validateOwnerID(formats strfmt.Registry) error { if err := validate.Required("ownerId", "body", m.OwnerID); err != nil { @@ -118,6 +146,15 @@ func (m *ObjectInfo) validateOwnerID(formats strfmt.Registry) error { return nil } +func (m *ObjectInfo) validatePayloadSize(formats strfmt.Registry) error { + + if err := validate.Required("payloadSize", "body", m.PayloadSize); err != nil { + return err + } + + return nil +} + // ContextValidate validate this object info based on the context it is used func (m *ObjectInfo) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error diff --git a/gen/restapi/embedded_spec.go b/gen/restapi/embedded_spec.go index 043a3ee..cf3a9d2 100644 --- a/gen/restapi/embedded_spec.go +++ b/gen/restapi/embedded_spec.go @@ -454,8 +454,29 @@ func init() { }, "/objects/{containerId}/{objectId}": { "get": { - "summary": "Get object info by address and", + "summary": "Get object info by address", "operationId": "getObjectInfo", + "parameters": [ + { + "type": "integer", + "name": "range-offset", + "in": "query" + }, + { + "minimum": 1, + "type": "integer", + "name": "range-length", + "in": "query" + }, + { + "maximum": 524288000, + "type": "integer", + "default": 4194304, + "description": "Max payload size (in bytes) that can be included in the response.\nIf the actual size is greater than this params the payload won't be included in the response.\n", + "name": "max-payload-size", + "in": "query" + } + ], "responses": { "200": { "description": "Object info", @@ -729,7 +750,9 @@ func init() { "containerId", "objectId", "ownerId", - "attributes" + "attributes", + "objectSize", + "payloadSize" ], "properties": { "attributes": { @@ -744,8 +767,20 @@ func init() { "objectId": { "type": "string" }, + "objectSize": { + "description": "Object full payload size", + "type": "integer" + }, "ownerId": { "type": "string" + }, + "payload": { + "description": "Base64 encoded object payload", + "type": "string" + }, + "payloadSize": { + "description": "Payload size in response", + "type": "integer" } }, "example": { @@ -1532,8 +1567,31 @@ func init() { }, "/objects/{containerId}/{objectId}": { "get": { - "summary": "Get object info by address and", + "summary": "Get object info by address", "operationId": "getObjectInfo", + "parameters": [ + { + "minimum": 0, + "type": "integer", + "name": "range-offset", + "in": "query" + }, + { + "minimum": 1, + "type": "integer", + "name": "range-length", + "in": "query" + }, + { + "maximum": 524288000, + "minimum": 0, + "type": "integer", + "default": 4194304, + "description": "Max payload size (in bytes) that can be included in the response.\nIf the actual size is greater than this params the payload won't be included in the response.\n", + "name": "max-payload-size", + "in": "query" + } + ], "responses": { "200": { "description": "Object info", @@ -1827,7 +1885,9 @@ func init() { "containerId", "objectId", "ownerId", - "attributes" + "attributes", + "objectSize", + "payloadSize" ], "properties": { "attributes": { @@ -1842,8 +1902,20 @@ func init() { "objectId": { "type": "string" }, + "objectSize": { + "description": "Object full payload size", + "type": "integer" + }, "ownerId": { "type": "string" + }, + "payload": { + "description": "Base64 encoded object payload", + "type": "string" + }, + "payloadSize": { + "description": "Payload size in response", + "type": "integer" } }, "example": { diff --git a/gen/restapi/operations/get_object_info.go b/gen/restapi/operations/get_object_info.go index b617801..bd04e27 100644 --- a/gen/restapi/operations/get_object_info.go +++ b/gen/restapi/operations/get_object_info.go @@ -33,7 +33,7 @@ func NewGetObjectInfo(ctx *middleware.Context, handler GetObjectInfoHandler) *Ge /* GetObjectInfo swagger:route GET /objects/{containerId}/{objectId} getObjectInfo -Get object info by address and +Get object info by address */ type GetObjectInfo struct { diff --git a/gen/restapi/operations/get_object_info_parameters.go b/gen/restapi/operations/get_object_info_parameters.go index 3e091c0..e904089 100644 --- a/gen/restapi/operations/get_object_info_parameters.go +++ b/gen/restapi/operations/get_object_info_parameters.go @@ -23,10 +23,14 @@ func NewGetObjectInfoParams() GetObjectInfoParams { var ( // initialize parameters with default values + maxPayloadSizeDefault = int64(4.194304e+06) + walletConnectDefault = bool(false) ) return GetObjectInfoParams{ + MaxPayloadSize: &maxPayloadSizeDefault, + WalletConnect: &walletConnectDefault, } } @@ -55,11 +59,30 @@ type GetObjectInfoParams struct { In: path */ ContainerID string + /*Max payload size (in bytes) that can be included in the response. + If the actual size is greater than this params the payload won't be included in the response. + + Maximum: 5.24288e+08 + Minimum: 0 + In: query + Default: 4.194304e+06 + */ + MaxPayloadSize *int64 /*Base58 encoded object id Required: true In: path */ ObjectID string + /* + Minimum: 1 + In: query + */ + RangeLength *int64 + /* + Minimum: 0 + In: query + */ + RangeOffset *int64 /*Use wallect connect signature scheme or not In: query Default: false @@ -91,11 +114,26 @@ func (o *GetObjectInfoParams) BindRequest(r *http.Request, route *middleware.Mat res = append(res, err) } + qMaxPayloadSize, qhkMaxPayloadSize, _ := qs.GetOK("max-payload-size") + if err := o.bindMaxPayloadSize(qMaxPayloadSize, qhkMaxPayloadSize, route.Formats); err != nil { + res = append(res, err) + } + rObjectID, rhkObjectID, _ := route.Params.GetOK("objectId") if err := o.bindObjectID(rObjectID, rhkObjectID, route.Formats); err != nil { res = append(res, err) } + qRangeLength, qhkRangeLength, _ := qs.GetOK("range-length") + if err := o.bindRangeLength(qRangeLength, qhkRangeLength, route.Formats); err != nil { + res = append(res, err) + } + + qRangeOffset, qhkRangeOffset, _ := qs.GetOK("range-offset") + if err := o.bindRangeOffset(qRangeOffset, qhkRangeOffset, route.Formats); err != nil { + res = append(res, err) + } + qWalletConnect, qhkWalletConnect, _ := qs.GetOK("walletConnect") if err := o.bindWalletConnect(qWalletConnect, qhkWalletConnect, route.Formats); err != nil { res = append(res, err) @@ -160,6 +198,48 @@ func (o *GetObjectInfoParams) bindContainerID(rawData []string, hasKey bool, for return nil } +// bindMaxPayloadSize binds and validates parameter MaxPayloadSize from query. +func (o *GetObjectInfoParams) bindMaxPayloadSize(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: false + // AllowEmptyValue: false + + if raw == "" { // empty values pass all other validations + // Default values have been previously initialized by NewGetObjectInfoParams() + return nil + } + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("max-payload-size", "query", "int64", raw) + } + o.MaxPayloadSize = &value + + if err := o.validateMaxPayloadSize(formats); err != nil { + return err + } + + return nil +} + +// validateMaxPayloadSize carries on validations for parameter MaxPayloadSize +func (o *GetObjectInfoParams) validateMaxPayloadSize(formats strfmt.Registry) error { + + if err := validate.MinimumInt("max-payload-size", "query", *o.MaxPayloadSize, 0, false); err != nil { + return err + } + + if err := validate.MaximumInt("max-payload-size", "query", *o.MaxPayloadSize, 5.24288e+08, false); err != nil { + return err + } + + return nil +} + // bindObjectID binds and validates parameter ObjectID from path. func (o *GetObjectInfoParams) bindObjectID(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string @@ -174,6 +254,80 @@ func (o *GetObjectInfoParams) bindObjectID(rawData []string, hasKey bool, format return nil } +// bindRangeLength binds and validates parameter RangeLength from query. +func (o *GetObjectInfoParams) bindRangeLength(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: false + // AllowEmptyValue: false + + if raw == "" { // empty values pass all other validations + return nil + } + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("range-length", "query", "int64", raw) + } + o.RangeLength = &value + + if err := o.validateRangeLength(formats); err != nil { + return err + } + + return nil +} + +// validateRangeLength carries on validations for parameter RangeLength +func (o *GetObjectInfoParams) validateRangeLength(formats strfmt.Registry) error { + + if err := validate.MinimumInt("range-length", "query", *o.RangeLength, 1, false); err != nil { + return err + } + + return nil +} + +// bindRangeOffset binds and validates parameter RangeOffset from query. +func (o *GetObjectInfoParams) bindRangeOffset(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: false + // AllowEmptyValue: false + + if raw == "" { // empty values pass all other validations + return nil + } + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("range-offset", "query", "int64", raw) + } + o.RangeOffset = &value + + if err := o.validateRangeOffset(formats); err != nil { + return err + } + + return nil +} + +// validateRangeOffset carries on validations for parameter RangeOffset +func (o *GetObjectInfoParams) validateRangeOffset(formats strfmt.Registry) error { + + if err := validate.MinimumInt("range-offset", "query", *o.RangeOffset, 0, false); err != nil { + return err + } + + return nil +} + // bindWalletConnect binds and validates parameter WalletConnect from query. func (o *GetObjectInfoParams) bindWalletConnect(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string diff --git a/handlers/auth_test.go b/handlers/auth_test.go index 3cefe0b..bbcd149 100644 --- a/handlers/auth_test.go +++ b/handlers/auth_test.go @@ -61,6 +61,6 @@ func TestSign(t *testing.T) { Key: pubKeyHex, } - _, err = prepareBearerToken(bt) + _, err = prepareBearerToken(bt, false) require.NoError(t, err) } diff --git a/handlers/objects.go b/handlers/objects.go index d7778e4..f991ab2 100644 --- a/handlers/objects.go +++ b/handlers/objects.go @@ -5,7 +5,6 @@ import ( "crypto/ecdsa" "encoding/base64" "fmt" - "github.com/go-openapi/runtime/middleware" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-api-go/v2/acl" @@ -20,6 +19,8 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/token" "go.uber.org/zap" + "io" + "strings" ) // PutObjects handler that uploads object to NeoFS. @@ -88,6 +89,7 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo btoken, err := getBearerToken(principal, params.XBearerSignature, params.XBearerSignatureKey, *params.WalletConnect) if err != nil { + a.log.Error("get bearer token", zap.Error(err)) return errorResponse.WithPayload(NewError(err)) } @@ -97,6 +99,7 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo objInfo, err := a.pool.HeadObject(ctx, prm) if err != nil { + a.log.Error("head object", zap.Error(err)) return errorResponse.WithPayload(NewError(err)) } @@ -105,6 +108,8 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo resp.ObjectID = NewString(params.ObjectID) resp.OwnerID = NewString(objInfo.OwnerID().String()) resp.Attributes = make([]*models.Attribute, len(objInfo.Attributes())) + resp.ObjectSize = NewInteger(int64(objInfo.PayloadSize())) + resp.PayloadSize = NewInteger(0) for i, attr := range objInfo.Attributes() { resp.Attributes[i] = &models.Attribute{ @@ -113,6 +118,55 @@ func (a *API) GetObjectInfo(params operations.GetObjectInfoParams, principal *mo } } + var prmRange pool.PrmObjectRange + prmRange.SetAddress(*addr) + prmRange.UseBearer(btoken) + + var offset, length uint64 + if params.RangeOffset != nil || params.RangeLength != nil { + if params.RangeOffset == nil || params.RangeLength == nil { + a.log.Error("both offset and length must be provided") + return errorResponse.WithPayload(NewError(fmt.Errorf("both offset and length must be provided"))) + } + offset = uint64(*params.RangeOffset) + length = uint64(*params.RangeLength) + } else { + length = objInfo.PayloadSize() + } + prmRange.SetOffset(offset) + prmRange.SetLength(length) + + if uint64(*params.MaxPayloadSize) < length { + return operations.NewGetObjectInfoOK().WithPayload(&resp) + } + + rangeRes, err := a.pool.ObjectRange(ctx, prmRange) + if err != nil { + a.log.Error("range object", zap.Error(err)) + return errorResponse.WithPayload(NewError(err)) + } + + defer func() { + if err = rangeRes.Close(); err != nil { + a.log.Error("close range result", zap.Error(err)) + } + }() + + sb := new(strings.Builder) + encoder := base64.NewEncoder(base64.StdEncoding, sb) + payloadSize, err := io.Copy(encoder, rangeRes) + if err != nil { + a.log.Error("encode object payload", zap.Error(err)) + return errorResponse.WithPayload(NewError(err)) + } + if err = encoder.Close(); err != nil { + a.log.Error("close encoder", zap.Error(err)) + return errorResponse.WithPayload(NewError(err)) + } + + resp.Payload = sb.String() + resp.PayloadSize = NewInteger(payloadSize) + return operations.NewGetObjectInfoOK().WithPayload(&resp) } diff --git a/spec/rest.yaml b/spec/rest.yaml index dec31fc..4658933 100644 --- a/spec/rest.yaml +++ b/spec/rest.yaml @@ -184,7 +184,25 @@ paths: - $ref: '#/parameters/objectId' get: operationId: getObjectInfo - summary: Get object info by address and + summary: Get object info by address + parameters: + - in: query + name: range-offset + type: integer + minimum: 0 + - in: query + name: range-length + type: integer + minimum: 1 + - in: query + name: max-payload-size + type: integer + default: 4194304 + minimum: 0 + maximum: 524288000 + description: | + Max payload size (in bytes) that can be included in the response. + If the actual size is greater than this params the payload won't be included in the response. responses: 200: description: Object info @@ -613,11 +631,22 @@ definitions: type: array items: $ref: '#/definitions/Attribute' + objectSize: + type: integer + description: Object full payload size + payloadSize: + type: integer + description: Payload size in response + payload: + type: string + description: Base64 encoded object payload required: - containerId - objectId - ownerId - attributes + - objectSize + - payloadSize example: containerId: 5HZTn5qkRnmgSz9gSrw22CEdPPk6nQhkwf2Mgzyvkikv objectId: 8N3o7Dtr6T1xteCt6eRwhpmJ7JhME58Hyu1dvaswuTDd @@ -633,7 +662,7 @@ definitions: containerId: type: string objectId: - type: string + type: string required: - containerId - objectId