From b5650456493de7d4c6ded4ce7bd3eb96b1835c98 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 30 Jun 2021 12:29:43 +0300 Subject: [PATCH 1/3] [#93] Fixed order headers set Writing headers had no effect early. Signed-off-by: Denis Kirillov --- api/handler/get.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/handler/get.go b/api/handler/get.go index 65f0487d..93cc7f28 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -30,11 +30,13 @@ func newDetector(w io.Writer) *detector { func (d *detector) Write(data []byte) (int, error) { d.Once.Do(func() { + d.contentType = http.DetectContentType(data) if rw, ok := d.Writer.(http.ResponseWriter); ok { rw.WriteHeader(http.StatusOK) + if len(rw.Header().Get(api.ContentType)) == 0 { + rw.Header().Set(api.ContentType, d.contentType) + } } - - d.contentType = http.DetectContentType(data) }) return d.Writer.Write(data) @@ -136,15 +138,14 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { writeError(w, r, h.log, "could not parse range header", rid, bkt, obj, err) return } - if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil { - writeError(w, r, h.log, "could not get object", rid, bkt, obj, err) - return - } - writeHeaders(w.Header(), inf) if params != nil { writeRangeHeaders(w, params, inf.Size) } + if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil { + writeError(w, r, h.log, "could not get object", rid, bkt, obj, err) + return + } } func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) { From a6ec27b40d26a056979c9e271b1b3ba56dcdf283 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Jun 2021 16:40:26 +0300 Subject: [PATCH 2/3] [#93] Object ETag support Signed-off-by: Denis Kirillov --- api/handler/copy.go | 2 +- api/handler/get.go | 7 ++++--- api/handler/list.go | 4 ++-- api/handler/put.go | 14 ++++++++------ api/layer/object.go | 9 +++++++++ api/layer/util.go | 2 ++ 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/api/handler/copy.go b/api/handler/copy.go index 3468159a..460c93e7 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -81,7 +81,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { }, r.URL) return - } else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: inf.Created.Format(time.RFC3339)}); err != nil { + } else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: inf.Created.Format(time.RFC3339), ETag: inf.HashSum}); err != nil { h.log.Error("something went wrong", zap.String("request_id", rid), zap.String("dst_bucket_name", bkt), diff --git a/api/handler/get.go b/api/handler/get.go index 93cc7f28..7e032221 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -109,9 +109,10 @@ func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, } func writeHeaders(h http.Header, info *layer.ObjectInfo) { - h.Set("Content-Type", info.ContentType) - h.Set("Last-Modified", info.Created.Format(http.TimeFormat)) - h.Set("Content-Length", strconv.FormatInt(info.Size, 10)) + h.Set(api.ContentType, info.ContentType) + h.Set(api.LastModified, info.Created.Format(http.TimeFormat)) + h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) + h.Set(api.ETag, info.HashSum) for key, val := range info.Headers { h.Set("X-"+key, val) diff --git a/api/handler/list.go b/api/handler/list.go index a05257fb..3212f49b 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -196,7 +196,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes DisplayName: obj.Owner.String(), }, - // ETag: "", + ETag: obj.HashSum, // StorageClass: "", }) } @@ -259,7 +259,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R DisplayName: obj.Owner.String(), }, - // ETag: "", + ETag: obj.HashSum, // StorageClass: "", }) } diff --git a/api/handler/put.go b/api/handler/put.go index 2df676eb..cae8a8df 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -24,11 +24,12 @@ const ( func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { var ( - err error - req = mux.Vars(r) - bkt = req["bucket"] - obj = req["object"] - rid = api.GetRequestID(r.Context()) + err error + info *layer.ObjectInfo + req = mux.Vars(r) + bkt = req["bucket"] + obj = req["object"] + rid = api.GetRequestID(r.Context()) ) if _, err := h.obj.GetBucketInfo(r.Context(), bkt); err != nil { @@ -53,7 +54,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { Size: r.ContentLength, } - if _, err = h.obj.PutObject(r.Context(), params); err != nil { + if info, err = h.obj.PutObject(r.Context(), params); err != nil { h.log.Error("could not upload object", zap.String("request_id", rid), zap.String("bucket_name", bkt), @@ -69,6 +70,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { return } + w.Header().Set(api.ETag, info.HashSum) api.WriteSuccessResponseHeadersOnly(w) } diff --git a/api/layer/object.go b/api/layer/object.go index 738d2bd0..4fe56f92 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -142,6 +142,14 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return nil, err } + addr := object.NewAddress() + addr.SetObjectID(oid) + addr.SetContainerID(bkt.CID) + meta, err := n.objectHead(ctx, addr) + if err != nil { + return nil, err + } + return &ObjectInfo{ id: oid, @@ -152,6 +160,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, Created: time.Now(), Headers: p.Header, ContentType: r.contentType, + HashSum: meta.PayloadChecksum().String(), }, nil } diff --git a/api/layer/util.go b/api/layer/util.go index 90f9ddb4..7ef2faf5 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -22,6 +22,7 @@ type ( Size int64 ContentType string Created time.Time + HashSum string Owner *owner.ID Headers map[string]string } @@ -119,6 +120,7 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter Headers: userHeaders, Owner: meta.OwnerID(), Size: size, + HashSum: meta.PayloadChecksum().String(), } } From 2af1b16b596186c09df74a4ecd8d6065f810ff8c Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 30 Jun 2021 15:29:01 +0300 Subject: [PATCH 3/3] [#93] Removed unnecessary Signed-off-by: Denis Kirillov --- api/handler/get.go | 74 +++++++-------------------------------------- api/handler/head.go | 27 ++++++++++++++--- api/layer/util.go | 14 ++------- 3 files changed, 35 insertions(+), 80 deletions(-) diff --git a/api/handler/get.go b/api/handler/get.go index 7e032221..5113ad85 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -1,13 +1,10 @@ package handler import ( - "context" "fmt" - "io" "net/http" "strconv" "strings" - "sync" "github.com/gorilla/mux" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -15,63 +12,6 @@ import ( "go.uber.org/zap" ) -type ( - detector struct { - io.Writer - sync.Once - - contentType string - } -) - -func newDetector(w io.Writer) *detector { - return &detector{Writer: w} -} - -func (d *detector) Write(data []byte) (int, error) { - d.Once.Do(func() { - d.contentType = http.DetectContentType(data) - if rw, ok := d.Writer.(http.ResponseWriter); ok { - rw.WriteHeader(http.StatusOK) - if len(rw.Header().Get(api.ContentType)) == 0 { - rw.Header().Set(api.ContentType, d.contentType) - } - } - }) - - return d.Writer.Write(data) -} - -func (h *handler) contentTypeFetcher(ctx context.Context, w io.Writer, info *layer.ObjectInfo) (string, error) { - return h.contentTypeFetcherWithRange(ctx, w, info, nil) -} - -func (h *handler) contentTypeFetcherWithRange(ctx context.Context, w io.Writer, info *layer.ObjectInfo, rangeParams *layer.RangeParams) (string, error) { - if info.IsDir() { - if rangeParams != nil { - return "", fmt.Errorf("it is forbidden to request for a range in the directory") - } - return info.ContentType, nil - } - - writer := newDetector(w) - - params := &layer.GetObjectParams{ - Bucket: info.Bucket, - Object: info.Name, - Writer: writer, - Range: rangeParams, - } - - // params.Length = inf.Size - - if err := h.obj.GetObject(ctx, params); err != nil { - return "", err - } - - return writer.contentType, nil -} - func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) { const prefix = "bytes=" rangeHeader := headers.Get("Range") @@ -109,7 +49,9 @@ func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, } func writeHeaders(h http.Header, info *layer.ObjectInfo) { - h.Set(api.ContentType, info.ContentType) + if len(info.ContentType) > 0 { + h.Set(api.ContentType, info.ContentType) + } h.Set(api.LastModified, info.Created.Format(http.TimeFormat)) h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) h.Set(api.ETag, info.HashSum) @@ -143,9 +85,15 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { if params != nil { writeRangeHeaders(w, params, inf.Size) } - if inf.ContentType, err = h.contentTypeFetcherWithRange(r.Context(), w, inf, params); err != nil { + + getParams := &layer.GetObjectParams{ + Bucket: inf.Bucket, + Object: inf.Name, + Writer: w, + Range: params, + } + if err = h.obj.GetObject(r.Context(), getParams); err != nil { writeError(w, r, h.log, "could not get object", rid, bkt, obj, err) - return } } diff --git a/api/handler/head.go b/api/handler/head.go index 0b335dad..e3b75806 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "context" "net/http" @@ -12,10 +13,18 @@ import ( "google.golang.org/grpc/status" ) -type devNull int +const sizeToDetectType = 512 -func (d devNull) Write(p []byte) (n int, err error) { - return len(p), nil +func getRangeToDetectContentType(maxSize int64) *layer.RangeParams { + end := uint64(maxSize) + if sizeToDetectType < end { + end = sizeToDetectType + } + + return &layer.RangeParams{ + Start: 0, + End: end - 1, + } } func (h *handler) checkIsFolder(ctx context.Context, bucket, object string) *layer.ObjectInfo { @@ -73,7 +82,15 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { }, r.URL) return - } else if inf.ContentType, err = h.contentTypeFetcher(r.Context(), devNull(0), inf); err != nil { + } + buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType)) + getParams := &layer.GetObjectParams{ + Bucket: inf.Bucket, + Object: inf.Name, + Writer: buffer, + Range: getRangeToDetectContentType(inf.Size), + } + if err = h.obj.GetObject(r.Context(), getParams); err != nil { h.log.Error("could not get object", zap.String("request_id", rid), zap.String("bucket_name", bkt), @@ -89,7 +106,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { return } - + inf.ContentType = http.DetectContentType(buffer.Bytes()) writeHeaders(w.Header(), inf) w.WriteHeader(http.StatusOK) } diff --git a/api/layer/util.go b/api/layer/util.go index 7ef2faf5..1b1c70af 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -1,7 +1,6 @@ package layer import ( - "net/http" "os" "strconv" "strings" @@ -103,10 +102,10 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter filename = prefix + tail[:index+1] userHeaders = nil } else { - size, mimeType = getSizeAndMimeType(meta, mimeType) + size = int64(meta.PayloadSize()) } } else { - size, mimeType = getSizeAndMimeType(meta, mimeType) + size = int64(meta.PayloadSize()) } return &ObjectInfo{ @@ -124,15 +123,6 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter } } -func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) { - size = int64(meta.PayloadSize()) - mimeType = contentType - if len(mimeType) == 0 { - mimeType = http.DetectContentType(meta.Payload()) - } - return -} - func filenameFromObject(o *object.Object) string { var name = o.ID().String() for _, attr := range o.Attributes() {