From a455ec18c31921eb367ceae376d6db03b8262929 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Nov 2022 13:42:29 +0300 Subject: [PATCH] [#2007] services/object: Allocate memory on-demand in GET_RANGE For big objects we want to get OutOfRange error before all the memory is allocated. Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/services/object/internal/client/client.go | 23 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a41dba12..302e3696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for NeoFS Node - `neofs-cli container nodes`'s output (#1991) - Increase error counter for write-cache flush errors (#1818) - Correctly select the shard for applying tree service operations (#1996) +- Do not panic with bad inputs for `GET_RANGE` (#2007) ### Removed ### Updated diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 06b7461e..20e4c598 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -1,6 +1,7 @@ package internal import ( + "bytes" "context" "crypto/ecdsa" "errors" @@ -316,6 +317,11 @@ func (x PayloadRangeRes) PayloadRange() []byte { return x.data } +// maxInitialBufferSize is the maximum initial buffer size for PayloadRange result. +// We don't want to allocate a lot of space in advance because a query can +// fail with apistatus.ObjectOutOfRange status. +const maxInitialBufferSize = 1024 * 1024 // 1 MiB + // PayloadRange reads object payload range by address. // // Client, context and key must be set. @@ -349,15 +355,26 @@ func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { return nil, fmt.Errorf("init payload reading: %w", err) } - data := make([]byte, prm.ln) + if int64(prm.ln) < 0 { + // `CopyN` expects `int64`, this check ensures that the result is positive. + // On practice this means that we can return incorrect results for objects + // with size > 8_388 Petabytes, this will be fixed later with support for streaming. + return nil, apistatus.ObjectOutOfRange{} + } - _, err = io.ReadFull(rdr, data) + ln := prm.ln + if ln > maxInitialBufferSize { + ln = maxInitialBufferSize + } + + w := bytes.NewBuffer(make([]byte, ln)) + _, err = io.CopyN(w, rdr, int64(prm.ln)) if err != nil { return nil, fmt.Errorf("read payload: %w", err) } return &PayloadRangeRes{ - data: data, + data: w.Bytes(), }, nil }