alexvanin/frostfs-http-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-http-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#76] Refactor HeadObject logic

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-07-15 10:57:22 +03:00
parent d856fdf4c4
commit 87918483c6
3 changed files with 91 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -262,6 +262,8 @@ $ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYov
#### Requests
The following requests support GET/HEAD methods.
##### By IDs
Basic downloading involves container ID and object ID and is done via GET
@ -311,7 +313,7 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m
#### Replies
You get object contents in the reply body, but at the same time you also get a
You get object contents in the reply body (if GET method was used), but at the same time you also get a
set of reply headers generated using the following rules:
* `Content-Length` is set to the length of the object
* `Content-Type` is autodetected dynamically by gateway

75
downloader/download.go
View file

@ -47,8 +47,6 @@ type (
var errObjectNotFound = errors.New("object not found")
const sizeToDetectType = 512
func newReader(data []byte, err error) *errReader {
return &errReader{data: data, err: err}
}
@ -198,69 +196,6 @@ func bearerOpts(ctx context.Context) client.CallOption {
return client.WithBearer(nil)
}
func (r request) headObject(clnt client.Object, objectAddress *object.Address) {
var start = time.Now()
if err := tokens.StoreBearerToken(r.RequestCtx); err != nil {
r.log.Error("could not fetch and store bearer token", zap.Error(err))
r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest)
return
}
options := new(client.ObjectHeaderParams).WithAddress(objectAddress)
bearerOpt := bearerOpts(r.RequestCtx)
obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10))
var contentType string
for _, attr := range obj.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
continue
}
r.Response.Header.Set("X-Attribute-"+key, val)
switch key {
case object.AttributeTimestamp:
value, err := strconv.ParseInt(val, 10, 64)
if err != nil {
r.log.Info("couldn't parse creation date",
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
continue
}
r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat))
case object.AttributeContentType:
contentType = val
}
}
r.Response.Header.Set("x-object-id", obj.ID().String())
r.Response.Header.Set("x-owner-id", obj.OwnerID().String())
r.Response.Header.Set("x-container-id", obj.ContainerID().String())
if len(contentType) == 0 {
objRange := object.NewRange()
objRange.SetOffset(0)
if sizeToDetectType < obj.PayloadSize() {
objRange.SetLength(sizeToDetectType)
} else {
objRange.SetLength(obj.PayloadSize())
}
ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange)
data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
contentType = http.DetectContentType(data)
}
r.SetContentType(contentType)
}
func (r *request) handleNeoFSErr(err error, start time.Time) {
r.log.Error(
"could not receive object",
@ -320,11 +255,6 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
d.byAddress(c, request.receiveFile)
}
// HeadByAddress handles head requests using simple cid/oid format.
func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) {
d.byAddress(c, request.headObject)
}
// byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) {
@ -349,11 +279,6 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
d.byAttribute(c, request.receiveFile)
}
// HeadByAttribute handles attribute-based head requests.
func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) {
d.byAttribute(c, request.headObject)
}
// byAttribute is wrapper similar to byAddress.
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) {
var (

88
downloader/head.go Normal file
View file

@ -0,0 +1,88 @@
package downloader
import (
"net/http"
"strconv"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
const sizeToDetectType = 512
func (r request) headObject(clnt client.Object, objectAddress *object.Address) {
var start = time.Now()
if err := tokens.StoreBearerToken(r.RequestCtx); err != nil {
r.log.Error("could not fetch and store bearer token", zap.Error(err))
r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest)
return
}
options := new(client.ObjectHeaderParams).WithAddress(objectAddress)
bearerOpt := bearerOpts(r.RequestCtx)
obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10))
var contentType string
for _, attr := range obj.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
continue
}
r.Response.Header.Set("X-Attribute-"+key, val)
switch key {
case object.AttributeTimestamp:
value, err := strconv.ParseInt(val, 10, 64)
if err != nil {
r.log.Info("couldn't parse creation date",
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
continue
}
r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat))
case object.AttributeContentType:
contentType = val
}
}
r.Response.Header.Set("x-object-id", obj.ID().String())
r.Response.Header.Set("x-owner-id", obj.OwnerID().String())
r.Response.Header.Set("x-container-id", obj.ContainerID().String())
if len(contentType) == 0 {
objRange := object.NewRange()
objRange.SetOffset(0)
if sizeToDetectType < obj.PayloadSize() {
objRange.SetLength(sizeToDetectType)
} else {
objRange.SetLength(obj.PayloadSize())
}
ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange)
data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
contentType = http.DetectContentType(data)
}
r.SetContentType(contentType)
}
// HeadByAddress handles head requests using simple cid/oid format.
func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) {
d.byAddress(c, request.headObject)
}
// HeadByAttribute handles attribute-based head requests.
func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) {
d.byAttribute(c, request.headObject)
}