forked from TrueCloudLab/frostfs-s3-gw
Merge pull request #106 from KirillovDenis/feature/96-support_getObject_ranges
[#96] Support ranges for GetObject
This commit is contained in:
Kirillov Denis
GitHub
commit
4d2021344e
4 changed files with 151 additions and 33 deletions
|
|
@ -2,9 +2,11 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
|
@ -39,7 +41,14 @@ func (d *detector) Write(data []byte) (int, error) {
|
|||
}
|
||||
|
||||
func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *layer.ObjectInfo) (string, error) {
|
||||
return h.contentTypeFetcherWithRange(ctx, w, info, nil)
|
||||
}
|
||||
|
||||
func (h *handler) contentTypeFetcherWithRange(ctx context.Context, w io.Writer, info *layer.ObjectInfo, rangeParams *layer.RangeParams) (string, error) {
|
||||
if info.IsDir() {
|
||||
if rangeParams != nil {
|
||||
return "", fmt.Errorf("it is forbidden to request for a range in the directory")
|
||||
}
|
||||
return info.ContentType, nil
|
||||
}
|
||||
|
||||
|
|
@ -49,6 +58,7 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay
|
|||
Bucket: info.Bucket,
|
||||
Object: info.Name,
|
||||
Writer: writer,
|
||||
Range: rangeParams,
|
||||
}
|
||||
|
||||
// params.Length = inf.Size
|
||||
|
|
@ -60,6 +70,42 @@ func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *lay
|
|||
return writer.contentType, nil
|
||||
}
|
||||
|
||||
func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) {
|
||||
const prefix = "bytes="
|
||||
rangeHeader := headers.Get("Range")
|
||||
if len(rangeHeader) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if !strings.HasPrefix(rangeHeader, prefix) {
|
||||
return nil, fmt.Errorf("unknown unit in range header")
|
||||
}
|
||||
arr := strings.Split(strings.TrimPrefix(rangeHeader, prefix), "-")
|
||||
if len(arr) != 2 || (len(arr[0]) == 0 && len(arr[1]) == 0) {
|
||||
return nil, fmt.Errorf("unknown byte-range-set")
|
||||
}
|
||||
|
||||
var end, start uint64
|
||||
var err0, err1 error
|
||||
base, bitSize := 10, 64
|
||||
|
||||
if len(arr[0]) == 0 {
|
||||
end, err1 = strconv.ParseUint(arr[1], base, bitSize)
|
||||
start = fullSize - end
|
||||
end = fullSize - 1
|
||||
} else if len(arr[1]) == 0 {
|
||||
start, err0 = strconv.ParseUint(arr[0], base, bitSize)
|
||||
end = fullSize - 1
|
||||
} else {
|
||||
start, err0 = strconv.ParseUint(arr[0], base, bitSize)
|
||||
end, err1 = strconv.ParseUint(arr[1], base, bitSize)
|
||||
}
|
||||
|
||||
if err0 != nil || err1 != nil || start > end {
|
||||
return nil, fmt.Errorf("invalid Range header")
|
||||
}
|
||||
return &layer.RangeParams{Start: start, End: end}, nil
|
||||
}
|
||||
|
||||
func writeHeaders(h http.Header, info *layer.ObjectInfo) {
|
||||
h.Set("Content-Type", info.ContentType)
|
||||
h.Set("Last-Modified", info.Created.Format(http.TimeFormat))
|
||||
|
|
@ -72,8 +118,9 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) {
|
|||
|
||||
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
err error
|
||||
inf *layer.ObjectInfo
|
||||
err error
|
||||
inf *layer.ObjectInfo
|
||||
params *layer.RangeParams
|
||||
|
||||
req = mux.Vars(r)
|
||||
bkt = req["bucket"]
|
||||
|
|
@ -82,34 +129,40 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
)
|
||||
|
||||
if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil {
|
||||
h.log.Error("could not find object",
|
||||
zap.String("request_id", rid),
|
||||
zap.String("bucket_name", bkt),
|
||||
zap.String("object_name", obj),
|
||||
zap.Error(err))
|
||||
|
||||
api.WriteErrorResponse(r.Context(), w, api.Error{
|
||||
Code: api.GetAPIError(api.ErrInternalError).Code,
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
}, r.URL)
|
||||
|
||||
writeError(w, r, h.log, "could not find object", rid, bkt, obj, err)
|
||||
return
|
||||
} else if inf.ContentType, err = h.contentTypeFetcher(r.Context(), w, inf); err != nil {
|
||||
h.log.Error("could not get object",
|
||||
zap.String("request_id", rid),
|
||||
zap.String("bucket_name", bkt),
|
||||
zap.String("object_name", obj),
|
||||
zap.Error(err))
|
||||
|
||||
api.WriteErrorResponse(r.Context(), w, api.Error{
|
||||
Code: api.GetAPIError(api.ErrInternalError).Code,
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
}, r.URL)
|
||||
|
||||
}
|
||||
if params, err = fetchRangeHeader(r.Header, uint64(inf.Size)); err != nil {
|
||||
writeError(w, r, h.log, "could not parse range header", rid, bkt, obj, err)
|
||||
return
|
||||
}
|
||||
if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil {
|
||||
writeError(w, r, h.log, "could not get object", rid, bkt, obj, err)
|
||||
return
|
||||
}
|
||||
|
||||
writeHeaders(w.Header(), inf)
|
||||
if params != nil {
|
||||
writeRangeHeaders(w, params, inf.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) {
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, r *http.Request, log *zap.Logger, msg, rid, bkt, obj string, err error) {
|
||||
log.Error(msg,
|
||||
zap.String("request_id", rid),
|
||||
zap.String("bucket_name", bkt),
|
||||
zap.String("object_name", obj),
|
||||
zap.Error(err))
|
||||
|
||||
api.WriteErrorResponse(r.Context(), w, api.Error{
|
||||
Code: api.GetAPIError(api.ErrInternalError).Code,
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
}, r.URL)
|
||||
}
|
||||
|
|
|
|||
41
api/handler/get_test.go
Normal file
41
api/handler/get_test.go
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFetchRangeHeader(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
header string
|
||||
expected *layer.RangeParams
|
||||
fullSize uint64
|
||||
err bool
|
||||
}{
|
||||
{header: "bytes=0-256", expected: &layer.RangeParams{Start: 0, End: 256}, err: false},
|
||||
{header: "bytes=0-0", expected: &layer.RangeParams{Start: 0, End: 0}, err: false},
|
||||
{header: "bytes=0-", expected: &layer.RangeParams{Start: 0, End: 99}, fullSize: 100, err: false},
|
||||
{header: "bytes=-10", expected: &layer.RangeParams{Start: 90, End: 99}, fullSize: 100, err: false},
|
||||
{header: "", err: false},
|
||||
{header: "bytes=-1-256", err: true},
|
||||
{header: "bytes=256-0", err: true},
|
||||
{header: "bytes=string-0", err: true},
|
||||
{header: "bytes=0-string", err: true},
|
||||
{header: "bytes:0-256", err: true},
|
||||
{header: "bytes:-", err: true},
|
||||
} {
|
||||
h := make(http.Header)
|
||||
h.Add("Range", tc.header)
|
||||
params, err := fetchRangeHeader(h, tc.fullSize)
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
continue
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, params)
|
||||
}
|
||||
}
|
||||
|
|
@ -38,6 +38,7 @@ type (
|
|||
|
||||
// GetObjectParams stores object get request parameters.
|
||||
GetObjectParams struct {
|
||||
Range *RangeParams
|
||||
Bucket string
|
||||
Object string
|
||||
Offset int64
|
||||
|
|
@ -45,6 +46,12 @@ type (
|
|||
Writer io.Writer
|
||||
}
|
||||
|
||||
// RangeParams stores range header request parameters.
|
||||
RangeParams struct {
|
||||
Start uint64
|
||||
End uint64
|
||||
}
|
||||
|
||||
// PutObjectParams stores object put request parameters.
|
||||
PutObjectParams struct {
|
||||
Bucket string
|
||||
|
|
@ -274,14 +281,23 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
|
|||
addr.SetObjectID(oid)
|
||||
addr.SetContainerID(bkt.CID)
|
||||
|
||||
_, err = n.objectGet(ctx, &getParams{
|
||||
Writer: p.Writer,
|
||||
|
||||
params := &getParams{
|
||||
Writer: p.Writer,
|
||||
address: addr,
|
||||
offset: p.Offset,
|
||||
length: p.Length,
|
||||
}
|
||||
|
||||
offset: p.Offset,
|
||||
length: p.Length,
|
||||
})
|
||||
if p.Range != nil {
|
||||
objRange := object.NewRange()
|
||||
objRange.SetOffset(p.Range.Start)
|
||||
// Range header is inclusive
|
||||
objRange.SetLength(p.Range.End - p.Range.Start + 1)
|
||||
params.Range = objRange
|
||||
_, err = n.objectRange(ctx, params)
|
||||
} else {
|
||||
_, err = n.objectGet(ctx, params)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err)
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ type (
|
|||
|
||||
getParams struct {
|
||||
io.Writer
|
||||
*object.Range
|
||||
|
||||
offset int64
|
||||
length int64
|
||||
|
|
@ -73,6 +74,13 @@ func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, er
|
|||
return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx))
|
||||
}
|
||||
|
||||
// objectRange gets object range and writes it into provided io.Writer.
|
||||
func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
|
||||
w := newWriter(p.Writer, p.offset, p.length)
|
||||
ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range)
|
||||
return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx))
|
||||
}
|
||||
|
||||
// objectPut into NeoFS, took payload from io.Reader.
|
||||
func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) {
|
||||
var (
|
||||
|
|
|
|||
Loading…
Reference in a new issue