From c4fef3d948304052d833b2190c64d3e17ca30077 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 24 Jun 2021 14:10:00 +0300 Subject: [PATCH] [#96] Support ranges for GetObject Signed-off-by: Denis Kirillov --- api/handler/get.go | 107 ++++++++++++++++++++++++++++++---------- api/handler/get_test.go | 41 +++++++++++++++ api/layer/layer.go | 28 ++++++++--- api/layer/object.go | 8 +++ 4 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 api/handler/get_test.go diff --git a/api/handler/get.go b/api/handler/get.go index c602a9012..65f0487d2 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -2,9 +2,11 @@ package handler import ( "context" + "fmt" "io" "net/http" "strconv" + "strings" "sync" "github.com/gorilla/mux" @@ -39,7 +41,14 @@ func (d *detector) Write(data []byte) (int, error) { } func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *layer.ObjectInfo) (string, error) { + return h.contentTypeFetcherWithRange(ctx, w, info, nil) +} + +func (h *handler) contentTypeFetcherWithRange(ctx context.Context, w io.Writer, info *layer.ObjectInfo, rangeParams *layer.RangeParams) (string, error) { if info.IsDir() { + if rangeParams != nil { + return "", fmt.Errorf("it is forbidden to request for a range in the directory") + } return info.ContentType, nil } @@ -49,6 +58,7 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay Bucket: info.Bucket, Object: info.Name, Writer: writer, + Range: rangeParams, } // params.Length = inf.Size @@ -60,6 +70,42 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay return writer.contentType, nil } +func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) { + const prefix = "bytes=" + rangeHeader := headers.Get("Range") + if len(rangeHeader) == 0 { + return nil, nil + } + if !strings.HasPrefix(rangeHeader, prefix) { + return nil, fmt.Errorf("unknown unit in range header") + } + arr := strings.Split(strings.TrimPrefix(rangeHeader, prefix), "-") + if len(arr) != 2 || (len(arr[0]) == 0 && len(arr[1]) == 0) { + return nil, fmt.Errorf("unknown byte-range-set") + } + + var end, start uint64 + var err0, err1 error + base, bitSize := 10, 64 + + if len(arr[0]) == 0 { + end, err1 = strconv.ParseUint(arr[1], base, bitSize) + start = fullSize - end + end = fullSize - 1 + } else if len(arr[1]) == 0 { + start, err0 = strconv.ParseUint(arr[0], base, bitSize) + end = fullSize - 1 + } else { + start, err0 = strconv.ParseUint(arr[0], base, bitSize) + end, err1 = strconv.ParseUint(arr[1], base, bitSize) + } + + if err0 != nil || err1 != nil || start > end { + return nil, fmt.Errorf("invalid Range header") + } + return &layer.RangeParams{Start: start, End: end}, nil +} + func writeHeaders(h http.Header, info *layer.ObjectInfo) { h.Set("Content-Type", info.ContentType) h.Set("Last-Modified", info.Created.Format(http.TimeFormat)) @@ -72,8 +118,9 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) { func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { var ( - err error - inf *layer.ObjectInfo + err error + inf *layer.ObjectInfo + params *layer.RangeParams req = mux.Vars(r) bkt = req["bucket"] @@ -82,34 +129,40 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { ) if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil { - h.log.Error("could not find object", - zap.String("request_id", rid), - zap.String("bucket_name", bkt), - zap.String("object_name", obj), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - + writeError(w, r, h.log, "could not find object", rid, bkt, obj, err) return - } else if inf.ContentType, err = h.contentTypeFetcher(r.Context(), w, inf); err != nil { - h.log.Error("could not get object", - zap.String("request_id", rid), - zap.String("bucket_name", bkt), - zap.String("object_name", obj), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - + } + if params, err = fetchRangeHeader(r.Header, uint64(inf.Size)); err != nil { + writeError(w, r, h.log, "could not parse range header", rid, bkt, obj, err) + return + } + if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil { + writeError(w, r, h.log, "could not get object", rid, bkt, obj, err) return } writeHeaders(w.Header(), inf) + if params != nil { + writeRangeHeaders(w, params, inf.Size) + } +} + +func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) { + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size)) + w.WriteHeader(http.StatusPartialContent) +} + +func writeError(w http.ResponseWriter, r *http.Request, log *zap.Logger, msg, rid, bkt, obj string, err error) { + log.Error(msg, + zap.String("request_id", rid), + zap.String("bucket_name", bkt), + zap.String("object_name", obj), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrInternalError).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusInternalServerError, + }, r.URL) } diff --git a/api/handler/get_test.go b/api/handler/get_test.go new file mode 100644 index 000000000..69bc5bd9c --- /dev/null +++ b/api/handler/get_test.go @@ -0,0 +1,41 @@ +package handler + +import ( + "net/http" + "testing" + + "github.com/nspcc-dev/neofs-s3-gw/api/layer" + "github.com/stretchr/testify/require" +) + +func TestFetchRangeHeader(t *testing.T) { + for _, tc := range []struct { + header string + expected *layer.RangeParams + fullSize uint64 + err bool + }{ + {header: "bytes=0-256", expected: &layer.RangeParams{Start: 0, End: 256}, err: false}, + {header: "bytes=0-0", expected: &layer.RangeParams{Start: 0, End: 0}, err: false}, + {header: "bytes=0-", expected: &layer.RangeParams{Start: 0, End: 99}, fullSize: 100, err: false}, + {header: "bytes=-10", expected: &layer.RangeParams{Start: 90, End: 99}, fullSize: 100, err: false}, + {header: "", err: false}, + {header: "bytes=-1-256", err: true}, + {header: "bytes=256-0", err: true}, + {header: "bytes=string-0", err: true}, + {header: "bytes=0-string", err: true}, + {header: "bytes:0-256", err: true}, + {header: "bytes:-", err: true}, + } { + h := make(http.Header) + h.Add("Range", tc.header) + params, err := fetchRangeHeader(h, tc.fullSize) + if tc.err { + require.Error(t, err) + continue + } + + require.NoError(t, err) + require.Equal(t, tc.expected, params) + } +} diff --git a/api/layer/layer.go b/api/layer/layer.go index ff38fd625..a7c7c3bae 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -38,6 +38,7 @@ type ( // GetObjectParams stores object get request parameters. GetObjectParams struct { + Range *RangeParams Bucket string Object string Offset int64 @@ -45,6 +46,12 @@ type ( Writer io.Writer } + // RangeParams stores range header request parameters. + RangeParams struct { + Start uint64 + End uint64 + } + // PutObjectParams stores object put request parameters. PutObjectParams struct { Bucket string @@ -274,14 +281,23 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { addr.SetObjectID(oid) addr.SetContainerID(bkt.CID) - _, err = n.objectGet(ctx, &getParams{ - Writer: p.Writer, - + params := &getParams{ + Writer: p.Writer, address: addr, + offset: p.Offset, + length: p.Length, + } - offset: p.Offset, - length: p.Length, - }) + if p.Range != nil { + objRange := object.NewRange() + objRange.SetOffset(p.Range.Start) + // Range header is inclusive + objRange.SetLength(p.Range.End - p.Range.Start + 1) + params.Range = objRange + _, err = n.objectRange(ctx, params) + } else { + _, err = n.objectGet(ctx, params) + } if err != nil { return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err) diff --git a/api/layer/object.go b/api/layer/object.go index f17901460..738d2bd08 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -24,6 +24,7 @@ type ( getParams struct { io.Writer + *object.Range offset int64 length int64 @@ -73,6 +74,13 @@ func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, er return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx)) } +// objectRange gets object range and writes it into provided io.Writer. +func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { + w := newWriter(p.Writer, p.offset, p.length) + ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range) + return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx)) +} + // objectPut into NeoFS, took payload from io.Reader. func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) { var (