nzinkevich/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#96] Support ranges for GetObject

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-06-24 14:10:00 +03:00
parent 39a43c21a1
commit c4fef3d948
4 changed files with 151 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
api/handler/get.go
View file

@ -2,9 +2,11 @@ package handler
import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
"github.com/gorilla/mux"
@ -39,7 +41,14 @@ func (d *detector) Write(data []byte) (int, error) {
}
func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *layer.ObjectInfo) (string, error) {
return h.contentTypeFetcherWithRange(ctx, w, info, nil)
}
func (h *handler) contentTypeFetcherWithRange(ctx context.Context, w io.Writer, info *layer.ObjectInfo, rangeParams *layer.RangeParams) (string, error) {
if info.IsDir() {
if rangeParams != nil {
return "", fmt.Errorf("it is forbidden to request for a range in the directory")
}
return info.ContentType, nil
}
@ -49,6 +58,7 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay
Bucket: info.Bucket,
Object: info.Name,
Writer: writer,
Range: rangeParams,
}
// params.Length = inf.Size
@ -60,6 +70,42 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay
return writer.contentType, nil
}
func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) {
const prefix = "bytes="
rangeHeader := headers.Get("Range")
if len(rangeHeader) == 0 {
return nil, nil
}
if !strings.HasPrefix(rangeHeader, prefix) {
return nil, fmt.Errorf("unknown unit in range header")
}
arr := strings.Split(strings.TrimPrefix(rangeHeader, prefix), "-")
if len(arr) != 2 || (len(arr[0]) == 0 && len(arr[1]) == 0) {
return nil, fmt.Errorf("unknown byte-range-set")
}
var end, start uint64
var err0, err1 error
base, bitSize := 10, 64
if len(arr[0]) == 0 {
end, err1 = strconv.ParseUint(arr[1], base, bitSize)
start = fullSize - end
end = fullSize - 1
} else if len(arr[1]) == 0 {
start, err0 = strconv.ParseUint(arr[0], base, bitSize)
end = fullSize - 1
} else {
start, err0 = strconv.ParseUint(arr[0], base, bitSize)
end, err1 = strconv.ParseUint(arr[1], base, bitSize)
}
if err0 != nil || err1 != nil || start > end {
return nil, fmt.Errorf("invalid Range header")
}
return &layer.RangeParams{Start: start, End: end}, nil
}
func writeHeaders(h http.Header, info *layer.ObjectInfo) {
h.Set("Content-Type", info.ContentType)
h.Set("Last-Modified", info.Created.Format(http.TimeFormat))
@ -74,6 +120,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
inf *layer.ObjectInfo
params *layer.RangeParams
req = mux.Vars(r)
bkt = req["bucket"]
@ -82,34 +129,40 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
)
if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil {
h.log.Error("could not find object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
writeError(w, r, h.log, "could not find object", rid, bkt, obj, err)
return
} else if inf.ContentType, err = h.contentTypeFetcher(r.Context(), w, inf); err != nil {
h.log.Error("could not get object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
if params, err = fetchRangeHeader(r.Header, uint64(inf.Size)); err != nil {
writeError(w, r, h.log, "could not parse range header", rid, bkt, obj, err)
return
}
if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil {
writeError(w, r, h.log, "could not get object", rid, bkt, obj, err)
return
}
writeHeaders(w.Header(), inf)
if params != nil {
writeRangeHeaders(w, params, inf.Size)
}
}
func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) {
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size))
w.WriteHeader(http.StatusPartialContent)
}
func writeError(w http.ResponseWriter, r *http.Request, log *zap.Logger, msg, rid, bkt, obj string, err error) {
log.Error(msg,
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}

41
api/handler/get_test.go Normal file
View file

@ -0,0 +1,41 @@
package handler
import (
"net/http"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
func TestFetchRangeHeader(t *testing.T) {
for _, tc := range []struct {
header string
expected *layer.RangeParams
fullSize uint64
err bool
}{
{header: "bytes=0-256", expected: &layer.RangeParams{Start: 0, End: 256}, err: false},
{header: "bytes=0-0", expected: &layer.RangeParams{Start: 0, End: 0}, err: false},
{header: "bytes=0-", expected: &layer.RangeParams{Start: 0, End: 99}, fullSize: 100, err: false},
{header: "bytes=-10", expected: &layer.RangeParams{Start: 90, End: 99}, fullSize: 100, err: false},
{header: "", err: false},
{header: "bytes=-1-256", err: true},
{header: "bytes=256-0", err: true},
{header: "bytes=string-0", err: true},
{header: "bytes=0-string", err: true},
{header: "bytes:0-256", err: true},
{header: "bytes:-", err: true},
} {
h := make(http.Header)
h.Add("Range", tc.header)
params, err := fetchRangeHeader(h, tc.fullSize)
if tc.err {
require.Error(t, err)
continue
}
require.NoError(t, err)
require.Equal(t, tc.expected, params)
}
}

24
api/layer/layer.go
View file

@ -38,6 +38,7 @@ type (
// GetObjectParams stores object get request parameters.
GetObjectParams struct {
Range *RangeParams
Bucket string
Object string
Offset int64
@ -45,6 +46,12 @@ type (
Writer io.Writer
}
// RangeParams stores range header request parameters.
RangeParams struct {
Start uint64
End uint64
}
// PutObjectParams stores object put request parameters.
PutObjectParams struct {
Bucket string
@ -274,14 +281,23 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
addr.SetObjectID(oid)
addr.SetContainerID(bkt.CID)
_, err = n.objectGet(ctx, &getParams{
params := &getParams{
Writer: p.Writer,
address: addr,
offset: p.Offset,
length: p.Length,
})
}
if p.Range != nil {
objRange := object.NewRange()
objRange.SetOffset(p.Range.Start)
// Range header is inclusive
objRange.SetLength(p.Range.End - p.Range.Start + 1)
params.Range = objRange
_, err = n.objectRange(ctx, params)
} else {
_, err = n.objectGet(ctx, params)
}
if err != nil {
return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err)

8
api/layer/object.go
View file

@ -24,6 +24,7 @@ type (
getParams struct {
io.Writer
*object.Range
offset int64
length int64
@ -73,6 +74,13 @@ func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, er
return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx))
}
// objectRange gets object range and writes it into provided io.Writer.
func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
w := newWriter(p.Writer, p.offset, p.length)
ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range)
return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx))
}
// objectPut into NeoFS, took payload from io.Reader.
func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) {
var (