From 87918483c6c7da2fe322b3345fba6b972aef6e2d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 15 Jul 2021 10:57:22 +0300 Subject: [PATCH] [#76] Refactor HeadObject logic Signed-off-by: Denis Kirillov --- README.md | 4 +- downloader/download.go | 75 ----------------------------------- downloader/head.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 downloader/head.go diff --git a/README.md b/README.md index f4894da..57a1504 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,8 @@ $ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYov #### Requests +The following requests support GET/HEAD methods. + ##### By IDs Basic downloading involves container ID and object ID and is done via GET @@ -311,7 +313,7 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m #### Replies -You get object contents in the reply body, but at the same time you also get a +You get object contents in the reply body (if GET method was used), but at the same time you also get a set of reply headers generated using the following rules: * `Content-Length` is set to the length of the object * `Content-Type` is autodetected dynamically by gateway diff --git a/downloader/download.go b/downloader/download.go index 0b77969..87a1094 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -47,8 +47,6 @@ type ( var errObjectNotFound = errors.New("object not found") -const sizeToDetectType = 512 - func newReader(data []byte, err error) *errReader { return &errReader{data: data, err: err} } @@ -198,69 +196,6 @@ func bearerOpts(ctx context.Context) client.CallOption { return client.WithBearer(nil) } -func (r request) headObject(clnt client.Object, objectAddress *object.Address) { - var start = time.Now() - if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { - r.log.Error("could not fetch and store bearer token", zap.Error(err)) - r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) - return - } - - options := new(client.ObjectHeaderParams).WithAddress(objectAddress) - bearerOpt := bearerOpts(r.RequestCtx) - obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) - if err != nil { - r.handleNeoFSErr(err, start) - return - } - - r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) - var contentType string - for _, attr := range obj.Attributes() { - key := attr.Key() - val := attr.Value() - if !isValidToken(key) || !isValidValue(val) { - continue - } - r.Response.Header.Set("X-Attribute-"+key, val) - switch key { - case object.AttributeTimestamp: - value, err := strconv.ParseInt(val, 10, 64) - if err != nil { - r.log.Info("couldn't parse creation date", - zap.String("key", key), - zap.String("val", val), - zap.Error(err)) - continue - } - r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) - case object.AttributeContentType: - contentType = val - } - } - r.Response.Header.Set("x-object-id", obj.ID().String()) - r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) - r.Response.Header.Set("x-container-id", obj.ContainerID().String()) - - if len(contentType) == 0 { - objRange := object.NewRange() - objRange.SetOffset(0) - if sizeToDetectType < obj.PayloadSize() { - objRange.SetLength(sizeToDetectType) - } else { - objRange.SetLength(obj.PayloadSize()) - } - ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) - data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) - if err != nil { - r.handleNeoFSErr(err, start) - return - } - contentType = http.DetectContentType(data) - } - r.SetContentType(contentType) -} - func (r *request) handleNeoFSErr(err error, start time.Time) { r.log.Error( "could not receive object", @@ -320,11 +255,6 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { d.byAddress(c, request.receiveFile) } -// HeadByAddress handles head requests using simple cid/oid format. -func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { - d.byAddress(c, request.headObject) -} - // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { @@ -349,11 +279,6 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { d.byAttribute(c, request.receiveFile) } -// HeadByAttribute handles attribute-based head requests. -func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { - d.byAttribute(c, request.headObject) -} - // byAttribute is wrapper similar to byAddress. func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) { var ( diff --git a/downloader/head.go b/downloader/head.go new file mode 100644 index 0000000..5ecfd19 --- /dev/null +++ b/downloader/head.go @@ -0,0 +1,88 @@ +package downloader + +import ( + "net/http" + "strconv" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-http-gw/tokens" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +const sizeToDetectType = 512 + +func (r request) headObject(clnt client.Object, objectAddress *object.Address) { + var start = time.Now() + if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { + r.log.Error("could not fetch and store bearer token", zap.Error(err)) + r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + return + } + + options := new(client.ObjectHeaderParams).WithAddress(objectAddress) + bearerOpt := bearerOpts(r.RequestCtx) + obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + + r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10)) + var contentType string + for _, attr := range obj.Attributes() { + key := attr.Key() + val := attr.Value() + if !isValidToken(key) || !isValidValue(val) { + continue + } + r.Response.Header.Set("X-Attribute-"+key, val) + switch key { + case object.AttributeTimestamp: + value, err := strconv.ParseInt(val, 10, 64) + if err != nil { + r.log.Info("couldn't parse creation date", + zap.String("key", key), + zap.String("val", val), + zap.Error(err)) + continue + } + r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat)) + case object.AttributeContentType: + contentType = val + } + } + r.Response.Header.Set("x-object-id", obj.ID().String()) + r.Response.Header.Set("x-owner-id", obj.OwnerID().String()) + r.Response.Header.Set("x-container-id", obj.ContainerID().String()) + + if len(contentType) == 0 { + objRange := object.NewRange() + objRange.SetOffset(0) + if sizeToDetectType < obj.PayloadSize() { + objRange.SetLength(sizeToDetectType) + } else { + objRange.SetLength(obj.PayloadSize()) + } + ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) + data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + contentType = http.DetectContentType(data) + } + r.SetContentType(contentType) +} + +// HeadByAddress handles head requests using simple cid/oid format. +func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) { + d.byAddress(c, request.headObject) +} + +// HeadByAttribute handles attribute-based head requests. +func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) { + d.byAttribute(c, request.headObject) +}