From 8c93dbf867124a77cda742490fe9a3f3cceaa65a Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 22 Aug 2020 05:49:00 +0300 Subject: [PATCH] [#29] Implement ListObjectsV2 ListObjectsV1 and ListObjectsV2 now simplified and return different results closes #29 Signed-off-by: Evgeniy Kulikov --- api/handler/list.go | 89 +++++++++++++++++++++++++++++++++--- api/handler/response.go | 31 +++++++++++++ api/handler/unimplemented.go | 16 ------- 3 files changed, 113 insertions(+), 23 deletions(-) diff --git a/api/handler/list.go b/api/handler/list.go index 78b8b4a0..10c983f5 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -86,11 +86,10 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { } } -func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { +func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) { var ( err error arg *listObjectsArgs - res *ListObjectsResponse rid = api.GetRequestID(r.Context()) ) @@ -105,7 +104,7 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { HTTPStatusCode: http.StatusBadRequest, }, r.URL) - return + return nil, nil, err } list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{ @@ -125,10 +124,32 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { HTTPStatusCode: http.StatusInternalServerError, }, r.URL) - return + return nil, nil, err } - res = &ListObjectsResponse{ + return arg, list, nil +} + +func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { + var rid = api.GetRequestID(r.Context()) + if arg, list, err := h.listObjects(w, r); err != nil { + // error already sent to client + return + } else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil { + h.log.Error("something went wrong", + zap.String("request_id", rid), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrInternalError).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusInternalServerError, + }, r.URL) + } +} + +func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse { + res := &ListObjectsResponse{ Name: arg.Bucket, EncodingType: arg.Encode, Marker: arg.Marker, @@ -155,13 +176,25 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { UserMetadata: obj.Headers, LastModified: obj.Created.Format(time.RFC3339), + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + // ETag: "", - // Owner: Owner{}, // StorageClass: "", }) } - if err := api.EncodeToResponse(w, res); err != nil { + return res +} + +func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { + var rid = api.GetRequestID(r.Context()) + if arg, list, err := h.listObjects(w, r); err != nil { + // error already sent to client + return + } else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil { h.log.Error("something went wrong", zap.String("request_id", rid), zap.Error(err)) @@ -174,6 +207,48 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { } } +func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response { + res := &ListObjectsV2Response{ + Name: arg.Bucket, + EncodingType: arg.Encode, + Prefix: arg.Prefix, + MaxKeys: arg.MaxKeys, + Delimiter: arg.Delimeter, + + IsTruncated: list.IsTruncated, + + ContinuationToken: arg.Marker, + NextContinuationToken: list.NextContinuationToken, + } + + // fill common prefixes + for i := range list.Prefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ + Prefix: list.Prefixes[i], + }) + } + + // fill contents + for _, obj := range list.Objects { + res.Contents = append(res.Contents, Object{ + Key: obj.Name, + Size: obj.Size, + UserMetadata: obj.Headers, + LastModified: obj.Created.Format(time.RFC3339), + + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + + // ETag: "", + // StorageClass: "", + }) + } + + return res +} + func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { var ( err error diff --git a/api/handler/response.go b/api/handler/response.go index d29d263f..7ce57b99 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -14,6 +14,37 @@ type ListBucketsResponse struct { } // Buckets are nested } +// ListObjectsV2Response - format for list objects response. +type ListObjectsV2Response struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + Name string + Prefix string + StartAfter string `xml:"StartAfter,omitempty"` + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + + KeyCount int + MaxKeys int + Delimiter string + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + Contents []Object + CommonPrefixes []CommonPrefix + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + // Bucket container for bucket metadata type Bucket struct { Name string diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index 378efc05..2525549d 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -295,14 +295,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) }, r.URL) } -func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) -} - func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { api.WriteErrorResponse(r.Context(), w, api.Error{ Code: "XNeoFSUnimplemented", @@ -343,14 +335,6 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt }, r.URL) } -func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) -} - func (h *handler) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { api.WriteErrorResponse(r.Context(), w, api.Error{ Code: "XNeoFSUnimplemented",