From da4eca5da56b109b91452ef013456b883034c258 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 30 Jun 2021 18:10:11 +0300 Subject: [PATCH 1/4] [#94] GetObject support conditional headers Supported If-Modified-Since and If-Unmodified-Since. Signed-off-by: Denis Kirillov --- api/handler/get.go | 42 ++++++++++++++++++++++++++++++++++++++++++ api/headers.go | 4 +++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/api/handler/get.go b/api/handler/get.go index 5113ad8..2450062 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -5,6 +5,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/gorilla/mux" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -12,6 +13,11 @@ import ( "go.uber.org/zap" ) +type getObjectArgs struct { + IfModifiedSince time.Time + IfUnmodifiedSince time.Time +} + func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) { const prefix = "bytes=" rangeHeader := headers.Get("Range") @@ -73,10 +79,25 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { rid = api.GetRequestID(r.Context()) ) + args, err := parseGetObjectArgs(r.Header) + if err != nil { + return + } + if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil { writeError(w, r, h.log, "could not find object", rid, bkt, obj, err) return } + + if !args.IfModifiedSince.IsZero() && inf.Created.Before(args.IfModifiedSince) { + w.WriteHeader(http.StatusNotModified) + return + } + if !args.IfUnmodifiedSince.IsZero() && inf.Created.After(args.IfUnmodifiedSince) { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + if params, err = fetchRangeHeader(r.Header, uint64(inf.Size)); err != nil { writeError(w, r, h.log, "could not parse range header", rid, bkt, obj, err) return @@ -97,6 +118,27 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } } +func parseGetObjectArgs(headers http.Header) (*getObjectArgs, error) { + var err error + args := &getObjectArgs{} + + ifModifiedSince := headers.Get(api.IfModifiedSince) + if len(ifModifiedSince) > 0 { + if args.IfModifiedSince, err = time.Parse(http.TimeFormat, ifModifiedSince); err != nil { + return nil, fmt.Errorf("couldn't parse %s header: %w", api.IfModifiedSince, err) + } + } + + ifUnmodifiedSince := headers.Get(api.IfUnmodifiedSince) + if len(ifUnmodifiedSince) > 0 { + if args.IfUnmodifiedSince, err = time.Parse(http.TimeFormat, ifUnmodifiedSince); err != nil { + return nil, fmt.Errorf("couldn't parse %s header: %w", api.IfUnmodifiedSince, err) + } + } + + return args, nil +} + func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) { w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size)) diff --git a/api/headers.go b/api/headers.go index 085e3f7..4e8380f 100644 --- a/api/headers.go +++ b/api/headers.go @@ -1,6 +1,6 @@ package api -// Standard S3 HTTP response constants. +// Standard S3 HTTP request/response constants. const ( LastModified = "Last-Modified" Date = "Date" @@ -22,4 +22,6 @@ const ( ContentDisposition = "Content-Disposition" Authorization = "Authorization" Action = "Action" + IfModifiedSince = "If-Modified-Since" + IfUnmodifiedSince = "If-Unmodified-Since" ) From ab8dd4201c25e8bd5ac0a62dbefe076043b5a731 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 1 Jul 2021 10:45:55 +0300 Subject: [PATCH 2/4] [#94] CopyObject support conditional headers Supported X-Amz-Copy-Source-If-Modified-Since and X-Amz-Copy-Source-If-Unmodified-Since. Signed-off-by: Denis Kirillov --- api/handler/copy.go | 87 ++++++++++++++++++++++++++++++--------------- api/handler/get.go | 30 ++++++++++------ api/headers.go | 3 ++ api/layer/layer.go | 15 ++------ 4 files changed, 84 insertions(+), 51 deletions(-) diff --git a/api/handler/copy.go b/api/handler/copy.go index 460c93e..aa6009d 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -12,6 +12,11 @@ import ( "go.uber.org/zap" ) +type copyObjectArgs struct { + IfModifiedSince time.Time + IfUnmodifiedSince time.Time +} + // path2BucketObject returns bucket and object. func path2BucketObject(path string) (bucket, prefix string) { path = strings.TrimPrefix(path, api.SlashSeparator) @@ -58,44 +63,40 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { srcBucket, srcObject := path2BucketObject(src) + args, err := parseCopyObjectArgs(r.Header) + if err != nil { + writeError(w, r, h.log, "could not parse request params ", rid, bkt, obj, err) + return + } + + if inf, err = h.obj.GetObjectInfo(r.Context(), srcBucket, srcObject); err != nil { + writeError(w, r, h.log, "could not find object", rid, bkt, obj, err) + return + } + + if !args.IfModifiedSince.IsZero() && inf.Created.Before(args.IfModifiedSince) { + w.WriteHeader(http.StatusNotModified) + return + } + if !args.IfUnmodifiedSince.IsZero() && inf.Created.After(args.IfUnmodifiedSince) { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + params := &layer.CopyObjectParams{ SrcBucket: srcBucket, DstBucket: bkt, SrcObject: srcObject, DstObject: obj, + SrcSize: inf.Size, + Header: inf.Headers, } if inf, err = h.obj.CopyObject(r.Context(), params); err != nil { - h.log.Error("could not copy object", - zap.String("request_id", rid), - zap.String("dst_bucket_name", bkt), - zap.String("dst_object_name", obj), - zap.String("src_bucket_name", srcBucket), - zap.String("src_object_name", srcObject), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - + writeErrorCopy(w, r, h.log, "could not copy object", rid, bkt, obj, srcBucket, srcObject, err) return } else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: inf.Created.Format(time.RFC3339), ETag: inf.HashSum}); err != nil { - h.log.Error("something went wrong", - zap.String("request_id", rid), - zap.String("dst_bucket_name", bkt), - zap.String("dst_object_name", obj), - zap.String("src_bucket_name", srcBucket), - zap.String("src_object_name", srcObject), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - + writeErrorCopy(w, r, h.log, "something went wrong", rid, bkt, obj, srcBucket, srcObject, err) return } @@ -104,3 +105,33 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { zap.String("object", inf.Name), zap.Stringer("object_id", inf.ID())) } + +func writeErrorCopy(w http.ResponseWriter, r *http.Request, log *zap.Logger, msg, rid, bkt, obj, srcBucket, srcObject string, err error) { + log.Error(msg, + zap.String("request_id", rid), + zap.String("dst_bucket_name", bkt), + zap.String("dst_object_name", obj), + zap.String("src_bucket_name", srcBucket), + zap.String("src_object_name", srcObject), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrInternalError).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusInternalServerError, + }, r.URL) +} + +func parseCopyObjectArgs(headers http.Header) (*copyObjectArgs, error) { + var err error + args := ©ObjectArgs{} + + if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.AmzCopyIfModifiedSince)); err != nil { + return nil, err + } + if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.AmzCopyIfUnmodifiedSince)); err != nil { + return nil, err + } + + return args, nil +} diff --git a/api/handler/get.go b/api/handler/get.go index 2450062..c431d09 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -81,6 +81,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { args, err := parseGetObjectArgs(r.Header) if err != nil { + writeError(w, r, h.log, "could not parse request params", rid, bkt, obj, err) return } @@ -122,23 +123,30 @@ func parseGetObjectArgs(headers http.Header) (*getObjectArgs, error) { var err error args := &getObjectArgs{} - ifModifiedSince := headers.Get(api.IfModifiedSince) - if len(ifModifiedSince) > 0 { - if args.IfModifiedSince, err = time.Parse(http.TimeFormat, ifModifiedSince); err != nil { - return nil, fmt.Errorf("couldn't parse %s header: %w", api.IfModifiedSince, err) - } + if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.IfModifiedSince)); err != nil { + return nil, err } - - ifUnmodifiedSince := headers.Get(api.IfUnmodifiedSince) - if len(ifUnmodifiedSince) > 0 { - if args.IfUnmodifiedSince, err = time.Parse(http.TimeFormat, ifUnmodifiedSince); err != nil { - return nil, fmt.Errorf("couldn't parse %s header: %w", api.IfUnmodifiedSince, err) - } + if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err != nil { + return nil, err } return args, nil } +func parseHTTPTime(data string) (time.Time, error) { + var result time.Time + var err error + + if len(data) == 0 { + return result, nil + } + + if result, err = time.Parse(http.TimeFormat, data); err != nil { + return result, fmt.Errorf("couldn't parse http time %s: %w", data, err) + } + return result, nil +} + func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) { w.Header().Set("Accept-Ranges", "bytes") w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", params.Start, params.End, size)) diff --git a/api/headers.go b/api/headers.go index 4e8380f..735ebb6 100644 --- a/api/headers.go +++ b/api/headers.go @@ -24,4 +24,7 @@ const ( Action = "Action" IfModifiedSince = "If-Modified-Since" IfUnmodifiedSince = "If-Unmodified-Since" + + AmzCopyIfModifiedSince = "X-Amz-Copy-Source-If-Modified-Since" + AmzCopyIfUnmodifiedSince = "X-Amz-Copy-Source-If-Unmodified-Since" ) diff --git a/api/layer/layer.go b/api/layer/layer.go index 30593e5..563ba8f 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -68,6 +68,7 @@ type ( DstBucket string SrcObject string DstObject string + SrcSize int64 Header map[string]string } // CreateBucketParams stores bucket create request parameters. @@ -378,11 +379,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, // CopyObject from one bucket into another bucket. func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) { - info, err := n.GetObjectInfo(ctx, p.SrcBucket, p.SrcObject) - if err != nil { - return nil, fmt.Errorf("couldn't get object info: %w", err) - } - pr, pw := io.Pipe() go func() { @@ -397,17 +393,12 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf } }() - // set custom headers - for k, v := range p.Header { - info.Headers[k] = v - } - return n.PutObject(ctx, &PutObjectParams{ Bucket: p.DstBucket, Object: p.DstObject, - Size: info.Size, + Size: p.SrcSize, Reader: pr, - Header: info.Headers, + Header: p.Header, }) } From c4631e5806b18b86879c632caa67cb878b036809 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 1 Jul 2021 14:24:00 +0300 Subject: [PATCH 3/4] [#94] Refactoring Signed-off-by: Denis Kirillov --- api/handler/copy.go | 8 ++++---- api/handler/get.go | 22 ++++++++++------------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/api/handler/copy.go b/api/handler/copy.go index aa6009d..67d5ebb 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -13,8 +13,8 @@ import ( ) type copyObjectArgs struct { - IfModifiedSince time.Time - IfUnmodifiedSince time.Time + IfModifiedSince *time.Time + IfUnmodifiedSince *time.Time } // path2BucketObject returns bucket and object. @@ -74,11 +74,11 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if !args.IfModifiedSince.IsZero() && inf.Created.Before(args.IfModifiedSince) { + if args.IfModifiedSince != nil && inf.Created.Before(*args.IfModifiedSince) { w.WriteHeader(http.StatusNotModified) return } - if !args.IfUnmodifiedSince.IsZero() && inf.Created.After(args.IfUnmodifiedSince) { + if args.IfUnmodifiedSince != nil && inf.Created.After(*args.IfUnmodifiedSince) { w.WriteHeader(http.StatusPreconditionFailed) return } diff --git a/api/handler/get.go b/api/handler/get.go index c431d09..eabacd1 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -14,8 +14,8 @@ import ( ) type getObjectArgs struct { - IfModifiedSince time.Time - IfUnmodifiedSince time.Time + IfModifiedSince *time.Time + IfUnmodifiedSince *time.Time } func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) { @@ -90,11 +90,11 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if !args.IfModifiedSince.IsZero() && inf.Created.Before(args.IfModifiedSince) { + if args.IfModifiedSince != nil && inf.Created.Before(*args.IfModifiedSince) { w.WriteHeader(http.StatusNotModified) return } - if !args.IfUnmodifiedSince.IsZero() && inf.Created.After(args.IfUnmodifiedSince) { + if args.IfUnmodifiedSince != nil && inf.Created.After(*args.IfUnmodifiedSince) { w.WriteHeader(http.StatusPreconditionFailed) return } @@ -133,18 +133,16 @@ func parseGetObjectArgs(headers http.Header) (*getObjectArgs, error) { return args, nil } -func parseHTTPTime(data string) (time.Time, error) { - var result time.Time - var err error - +func parseHTTPTime(data string) (*time.Time, error) { if len(data) == 0 { - return result, nil + return nil, nil } - if result, err = time.Parse(http.TimeFormat, data); err != nil { - return result, fmt.Errorf("couldn't parse http time %s: %w", data, err) + result, err := time.Parse(http.TimeFormat, data) + if err != nil { + return nil, fmt.Errorf("couldn't parse http time %s: %w", data, err) } - return result, nil + return &result, nil } func writeRangeHeaders(w http.ResponseWriter, params *layer.RangeParams, size int64) { From 116ffbb4384038f56a3055ea44d1a408d97b9a9a Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 1 Jul 2021 14:25:16 +0300 Subject: [PATCH 4/4] [#94] Fix time format Signed-off-by: Denis Kirillov --- api/handler/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/handler/get.go b/api/handler/get.go index eabacd1..1cb62fb 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -58,7 +58,7 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) { if len(info.ContentType) > 0 { h.Set(api.ContentType, info.ContentType) } - h.Set(api.LastModified, info.Created.Format(http.TimeFormat)) + h.Set(api.LastModified, info.Created.Format(time.RFC3339)) h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) h.Set(api.ETag, info.HashSum)