From ee840621548101b25602e6f16640487ad685f9ec Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Sun, 18 Jul 2021 16:40:19 +0300 Subject: [PATCH 1/2] [#154] api: refactor ListObjectsV1 and V2 Separate ListObject for different versions. Remove useless grouping of keys on V2. Signed-off-by: Angira Kekteeva --- api/handler/head.go | 10 +- api/handler/list.go | 309 +------------------------------------ api/handler/object_list.go | 278 +++++++++++++++++++++++++++++++++ api/handler/response.go | 89 ++++------- api/layer/container.go | 11 -- api/layer/layer.go | 127 +-------------- api/layer/layer_test.go | 44 ------ api/layer/object.go | 163 +++++++++++++++++++ api/layer/util.go | 33 ++-- 9 files changed, 493 insertions(+), 571 deletions(-) create mode 100644 api/handler/object_list.go delete mode 100644 api/layer/layer_test.go diff --git a/api/handler/head.go b/api/handler/head.go index c916b7d6..bf177aa9 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -32,9 +32,13 @@ func (h *handler) checkIsFolder(ctx context.Context, bucket, object string) *lay } _, dirname := layer.NameFromString(object) - params := &layer.ListObjectsParams{Bucket: bucket, Prefix: dirname, Delimiter: layer.PathSeparator} - - if list, err := h.obj.ListObjects(ctx, params); err == nil && len(list.Objects) > 0 { + params := &layer.ListObjectsParamsV1{ + ListObjectsParamsCommon: layer.ListObjectsParamsCommon{ + Bucket: bucket, + Prefix: dirname, + Delimiter: layer.PathSeparator, + }} + if list, err := h.obj.ListObjectsV1(ctx, params); err == nil && len(list.Objects) > 0 { return &layer.ObjectInfo{ Bucket: bucket, Name: object, diff --git a/api/handler/list.go b/api/handler/list.go index 9b68bcd4..3711c4f2 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -3,26 +3,13 @@ package handler import ( "encoding/xml" "net/http" - "strconv" "time" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" - "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" ) -type listObjectsArgs struct { - Bucket string - Delimiter string - Encode string - Marker string - StartAfter string - MaxKeys int - Prefix string - APIVersion int -} - // VersioningConfiguration contains VersioningConfiguration XML representation. type VersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` @@ -45,11 +32,7 @@ func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, e zap.String("request_id", rid), zap.Error(err)) - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrBadRequest).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusBadRequest, - }, r.URL) + api.WriteErrorResponse(r.Context(), w, err, r.URL) } // ListBucketsHandler handles bucket listing requests. @@ -107,207 +90,6 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { } } -func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) { - var ( - err error - arg *listObjectsArgs - rid = api.GetRequestID(r.Context()) - ) - - if arg, err = parseListObjectArgs(r); err != nil { - h.log.Error("something went wrong", - zap.String("request_id", rid), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, err, r.URL) - return nil, nil, err - } - - marker := arg.Marker - if arg.APIVersion == 2 { - marker = arg.StartAfter - } - - list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{ - Bucket: arg.Bucket, - Prefix: arg.Prefix, - MaxKeys: arg.MaxKeys, - Delimiter: arg.Delimiter, - Marker: marker, - Version: arg.APIVersion, - }) - if err != nil { - h.log.Error("something went wrong", - zap.String("request_id", rid), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - - return nil, nil, err - } - - return arg, list, nil -} - -// ListObjectsV1Handler handles objects listing requests for API version 1. -func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { - var rid = api.GetRequestID(r.Context()) - if arg, list, err := h.listObjects(w, r); err != nil { - // error already sent to client - return - } else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil { - h.log.Error("something went wrong", - zap.String("request_id", rid), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - } -} - -func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse { - res := &ListObjectsResponse{ - Name: arg.Bucket, - EncodingType: arg.Encode, - Marker: arg.Marker, - Prefix: arg.Prefix, - MaxKeys: arg.MaxKeys, - Delimiter: arg.Delimiter, - - IsTruncated: list.IsTruncated, - NextMarker: list.NextMarker, - } - - // fill common prefixes - for i := range list.Prefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ - Prefix: s3PathEncode(list.Prefixes[i], arg.Encode), - }) - } - - // fill contents - for _, obj := range list.Objects { - res.Contents = append(res.Contents, Object{ - Key: s3PathEncode(obj.Name, arg.Encode), - Size: obj.Size, - LastModified: obj.Created.Format(time.RFC3339), - - Owner: Owner{ - ID: obj.Owner.String(), - DisplayName: obj.Owner.String(), - }, - - ETag: obj.HashSum, - // StorageClass: "", - }) - } - - return res -} - -// ListObjectsV2Handler handles objects listing requests for API version 2. -func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { - var rid = api.GetRequestID(r.Context()) - if arg, list, err := h.listObjects(w, r); err != nil { - // error already sent to client - return - } else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil { - h.log.Error("something went wrong", - zap.String("request_id", rid), - zap.Error(err)) - - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: api.GetAPIError(api.ErrInternalError).Code, - Description: err.Error(), - HTTPStatusCode: http.StatusInternalServerError, - }, r.URL) - } -} - -func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response { - res := &ListObjectsV2Response{ - Name: arg.Bucket, - EncodingType: arg.Encode, - Prefix: s3PathEncode(arg.Prefix, arg.Encode), - KeyCount: len(list.Objects) + len(list.Prefixes), - MaxKeys: arg.MaxKeys, - Delimiter: s3PathEncode(arg.Delimiter, arg.Encode), - StartAfter: s3PathEncode(arg.StartAfter, arg.Encode), - - IsTruncated: list.IsTruncated, - - ContinuationToken: arg.Marker, - NextContinuationToken: list.NextContinuationToken, - } - - // fill common prefixes - for i := range list.Prefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ - Prefix: s3PathEncode(list.Prefixes[i], arg.Encode), - }) - } - - // fill contents - for _, obj := range list.Objects { - res.Contents = append(res.Contents, Object{ - Key: s3PathEncode(obj.Name, arg.Encode), - Size: obj.Size, - LastModified: obj.Created.Format(time.RFC3339), - - Owner: Owner{ - ID: obj.Owner.String(), - DisplayName: obj.Owner.String(), - }, - - ETag: obj.HashSum, - // StorageClass: "", - }) - } - - return res -} - -func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { - var ( - err error - res listObjectsArgs - ) - - if r.URL.Query().Get("max-keys") == "" { - res.MaxKeys = maxObjectList - } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 { - return nil, api.GetAPIError(api.ErrInvalidMaxKeys) - } - - res.Prefix = r.URL.Query().Get("prefix") - res.Marker = r.URL.Query().Get("marker") - res.Delimiter = r.URL.Query().Get("delimiter") - res.Encode = r.URL.Query().Get("encoding-type") - res.StartAfter = r.URL.Query().Get("start-after") - apiVersionStr := r.URL.Query().Get("list-type") - - res.APIVersion = 1 - if len(apiVersionStr) != 0 { - if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 { - return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException) - } - res.APIVersion = 2 - } - - if info := api.GetReqInfo(r.Context()); info != nil { - res.Bucket = info.BucketName - } - - return &res, nil -} - // GetBucketVersioningHandler implements bucket versioning getter handler. func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -351,92 +133,3 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req }, r.URL) } } - -func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { - p, err := parseListObjectVersionsRequest(r) - if err != nil { - h.registerAndSendError(w, r, err, "failed to parse request ") - return - } - - info, err := h.obj.ListObjectVersions(r.Context(), p) - if err != nil { - h.registerAndSendError(w, r, err, "something went wrong") - return - } - - response := encodeListObjectVersionsToResponse(info, p.Bucket) - if err := api.EncodeToResponse(w, response); err != nil { - h.registerAndSendError(w, r, err, "something went wrong") - } -} - -func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) { - var ( - err error - res layer.ListObjectVersionsParams - ) - - if r.URL.Query().Get("max-keys") == "" { - res.MaxKeys = maxObjectList - } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 { - return nil, api.GetAPIError(api.ErrInvalidMaxKeys) - } - - res.Prefix = r.URL.Query().Get("prefix") - res.KeyMarker = r.URL.Query().Get("marker") - res.Delimiter = r.URL.Query().Get("delimiter") - res.Encode = r.URL.Query().Get("encoding-type") - res.VersionIDMarker = r.URL.Query().Get("version-id-marker") - - if info := api.GetReqInfo(r.Context()); info != nil { - res.Bucket = info.BucketName - } - - return &res, nil -} - -func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse { - res := ListObjectsVersionsResponse{ - Name: bucketName, - IsTruncated: info.IsTruncated, - KeyMarker: info.KeyMarker, - NextKeyMarker: info.NextKeyMarker, - NextVersionIDMarker: info.NextVersionIDMarker, - VersionIDMarker: info.VersionIDMarker, - } - - for _, prefix := range info.CommonPrefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix}) - } - - for _, ver := range info.Version { - res.Version = append(res.Version, ObjectVersionResponse{ - IsLatest: ver.IsLatest, - Key: ver.Object.Name, - LastModified: ver.Object.Created.Format(time.RFC3339), - Owner: Owner{ - ID: ver.Object.Owner.String(), - DisplayName: ver.Object.Owner.String(), - }, - Size: ver.Object.Size, - VersionID: ver.VersionID, - ETag: ver.Object.HashSum, - }) - } - // this loop is not starting till versioning is not implemented - for _, del := range info.DeleteMarker { - res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ - IsLatest: del.IsLatest, - Key: del.Key, - LastModified: del.LastModified, - Owner: Owner{ - ID: del.Owner.String(), - DisplayName: del.Owner.String(), - }, - VersionID: del.VersionID, - }) - } - - return &res -} diff --git a/api/handler/object_list.go b/api/handler/object_list.go new file mode 100644 index 00000000..0e7bbb3a --- /dev/null +++ b/api/handler/object_list.go @@ -0,0 +1,278 @@ +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/layer" +) + +// ListObjectsV1Handler handles objects listing requests for API version 1. +func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { + params, err := parseListObjectsArgsV1(r) + if err != nil { + h.registerAndSendError(w, r, err, "failed to parse arguments") + return + } + + list, err := h.obj.ListObjectsV1(r.Context(), params) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + return + } + + err = api.EncodeToResponse(w, encodeV1(params, list)) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + } +} + +func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response { + res := &ListObjectsV1Response{ + Name: p.Bucket, + EncodingType: p.Encode, + Marker: p.Marker, + Prefix: p.Prefix, + MaxKeys: p.MaxKeys, + Delimiter: p.Delimiter, + + IsTruncated: list.IsTruncated, + NextMarker: list.NextMarker, + } + + // fill common prefixes + for i := range list.Prefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ + Prefix: s3PathEncode(list.Prefixes[i], p.Encode), + }) + } + + // fill contents + for _, obj := range list.Objects { + res.Contents = append(res.Contents, Object{ + Key: s3PathEncode(obj.Name, p.Encode), + Size: obj.Size, + LastModified: obj.Created.Format(time.RFC3339), + + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + + ETag: obj.HashSum, + }) + } + + return res +} + +// ListObjectsV2Handler handles objects listing requests for API version 2. +func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { + params, err := parseListObjectsArgsV2(r) + if err != nil { + h.registerAndSendError(w, r, err, "failed to parse arguments") + return + } + + list, err := h.obj.ListObjectsV2(r.Context(), params) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + return + } + + err = api.EncodeToResponse(w, encodeV2(params, list)) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + } +} + +func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response { + res := &ListObjectsV2Response{ + Name: p.Bucket, + EncodingType: p.Encode, + Prefix: s3PathEncode(p.Prefix, p.Encode), + KeyCount: len(list.Objects) + len(list.Prefixes), + MaxKeys: p.MaxKeys, + Delimiter: s3PathEncode(p.Delimiter, p.Encode), + StartAfter: s3PathEncode(p.StartAfter, p.Encode), + + IsTruncated: list.IsTruncated, + + ContinuationToken: p.ContinuationToken, + NextContinuationToken: list.NextContinuationToken, + } + + // fill common prefixes + for i := range list.Prefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ + Prefix: s3PathEncode(list.Prefixes[i], p.Encode), + }) + } + + // fill contents + for _, obj := range list.Objects { + res.Contents = append(res.Contents, Object{ + Key: s3PathEncode(obj.Name, p.Encode), + Size: obj.Size, + LastModified: obj.Created.Format(time.RFC3339), + + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + + ETag: obj.HashSum, + }) + } + + return res +} + +func parseListObjectsArgsV1(r *http.Request) (*layer.ListObjectsParamsV1, error) { + var ( + err error + res layer.ListObjectsParamsV1 + ) + + common, err := parseListObjectArgs(r) + if err != nil { + return nil, err + } + res.ListObjectsParamsCommon = *common + + res.Marker = r.URL.Query().Get("marker") + + return &res, nil +} + +func parseListObjectsArgsV2(r *http.Request) (*layer.ListObjectsParamsV2, error) { + var ( + err error + res layer.ListObjectsParamsV2 + ) + + common, err := parseListObjectArgs(r) + if err != nil { + return nil, err + } + res.ListObjectsParamsCommon = *common + + res.ContinuationToken = r.URL.Query().Get("continuation-token") + res.StartAfter = r.URL.Query().Get("start-after") + return &res, nil +} + +func parseListObjectArgs(r *http.Request) (*layer.ListObjectsParamsCommon, error) { + var ( + err error + res layer.ListObjectsParamsCommon + ) + + if info := api.GetReqInfo(r.Context()); info != nil { + res.Bucket = info.BucketName + } + + res.Delimiter = r.URL.Query().Get("delimiter") + res.Encode = r.URL.Query().Get("encoding-type") + + if r.URL.Query().Get("max-keys") == "" { + res.MaxKeys = maxObjectList + } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 { + return nil, api.GetAPIError(api.ErrInvalidMaxKeys) + } + + res.Prefix = r.URL.Query().Get("prefix") + + return &res, nil +} + +func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { + p, err := parseListObjectVersionsRequest(r) + if err != nil { + h.registerAndSendError(w, r, err, "failed to parse request ") + return + } + + info, err := h.obj.ListObjectVersions(r.Context(), p) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + return + } + + response := encodeListObjectVersionsToResponse(info, p.Bucket) + if err := api.EncodeToResponse(w, response); err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + } +} + +func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) { + var ( + err error + res layer.ListObjectVersionsParams + ) + + if r.URL.Query().Get("max-keys") == "" { + res.MaxKeys = maxObjectList + } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 { + return nil, api.GetAPIError(api.ErrInvalidMaxKeys) + } + + res.Prefix = r.URL.Query().Get("prefix") + res.KeyMarker = r.URL.Query().Get("marker") + res.Delimiter = r.URL.Query().Get("delimiter") + res.Encode = r.URL.Query().Get("encoding-type") + res.VersionIDMarker = r.URL.Query().Get("version-id-marker") + + if info := api.GetReqInfo(r.Context()); info != nil { + res.Bucket = info.BucketName + } + + return &res, nil +} + +func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse { + res := ListObjectsVersionsResponse{ + Name: bucketName, + IsTruncated: info.IsTruncated, + KeyMarker: info.KeyMarker, + NextKeyMarker: info.NextKeyMarker, + NextVersionIDMarker: info.NextVersionIDMarker, + VersionIDMarker: info.VersionIDMarker, + } + + for _, prefix := range info.CommonPrefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix}) + } + + for _, ver := range info.Version { + res.Version = append(res.Version, ObjectVersionResponse{ + IsLatest: ver.IsLatest, + Key: ver.Object.Name, + LastModified: ver.Object.Created.Format(time.RFC3339), + Owner: Owner{ + ID: ver.Object.Owner.String(), + DisplayName: ver.Object.Owner.String(), + }, + Size: ver.Object.Size, + VersionID: ver.VersionID, + ETag: ver.Object.HashSum, + }) + } + // this loop is not starting till versioning is not implemented + for _, del := range info.DeleteMarker { + res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ + IsLatest: del.IsLatest, + Key: del.Key, + LastModified: del.LastModified, + Owner: Owner{ + ID: del.Owner.String(), + DisplayName: del.Owner.String(), + }, + VersionID: del.VersionID, + }) + } + + return &res +} diff --git a/api/handler/response.go b/api/handler/response.go index b57a7a04..e6475090 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -14,35 +14,36 @@ type ListBucketsResponse struct { } // Buckets are nested } -// ListObjectsV2Response - format for list objects response. +// ListObjectsV1Response -- format for ListObjectsV1 response. +type ListObjectsV1Response struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` + Contents []Object `xml:"Contents"` + Delimiter string `xml:"Delimiter,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + NextMarker string `xml:"NextMarker,omitempty"` + Prefix string `xml:"Prefix"` +} + +// ListObjectsV2Response -- format for ListObjectsV2 response. type ListObjectsV2Response struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` - - Name string - Prefix string - StartAfter string `xml:"StartAfter,omitempty"` - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Server lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. - ContinuationToken string `xml:"ContinuationToken,omitempty"` - NextContinuationToken string `xml:"NextContinuationToken,omitempty"` - - KeyCount int - MaxKeys int - Delimiter string `xml:"Delimiter,omitempty"` - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - - Contents []Object - CommonPrefixes []CommonPrefix - - // Encoding type used to encode object keys in the response. - EncodingType string `xml:"EncodingType,omitempty"` + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` + Contents []Object `xml:"Contents"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + IsTruncated bool `xml:"IsTruncated"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + Prefix string `xml:"Prefix"` + StartAfter string `xml:"StartAfter,omitempty"` } // Bucket container for bucket metadata. @@ -57,37 +58,7 @@ type Owner struct { DisplayName string } -// ListObjectsResponse - format for list objects response. -type ListObjectsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` - - Name string - Prefix string - Marker string - - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Server lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. - NextMarker string `xml:"NextMarker,omitempty"` - - MaxKeys int - Delimiter string `xml:"Delimiter,omitempty"` - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - - Contents []Object - CommonPrefixes []CommonPrefix - - // Encoding type used to encode object keys in the response. - EncodingType string `xml:"EncodingType,omitempty"` -} - -// CommonPrefix container for prefix response in ListObjectsResponse. +// CommonPrefix container for prefix response in ListObjects's response. type CommonPrefix struct { Prefix string } diff --git a/api/layer/container.go b/api/layer/container.go index ce302327..c5619686 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -28,17 +28,6 @@ type ( Owner *owner.ID Created time.Time } - - // ListObjectsParams represents object listing request parameters. - ListObjectsParams struct { - Bucket string - Prefix string - Token string - Delimiter string - MaxKeys int - Marker string - Version int - } ) func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) { diff --git a/api/layer/layer.go b/api/layer/layer.go index 45f047e9..a3d348b9 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -8,7 +8,6 @@ import ( "io" "net/url" "sort" - "strings" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -114,7 +113,8 @@ type ( CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) - ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) + ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) + ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) DeleteObject(ctx context.Context, bucket, object string) error @@ -134,8 +134,6 @@ var ( ) const ( - // ETag (hex encoded md5sum) of empty string. - emptyETag = "d41d8cd98f00b204e9800998ecf8427e" unversionedObjectVersionID = "null" ) @@ -212,127 +210,6 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) { return n.containerList(ctx) } -// ListObjects returns objects from the container. It ignores tombstones and -// storage groups. -// ctx, bucket, prefix, continuationToken, delimiter, maxKeys -func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) { - // todo: make pagination when search response will be gRPC stream, - // pagination must be implemented with cache, because search results - // may be different between search calls - var ( - err error - bkt *BucketInfo - ids []*object.ID - result ListObjectsInfo - uniqNames = make(map[string]bool) - ) - - if p.MaxKeys == 0 { - return &result, nil - } - - if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - return nil, err - } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { - return nil, err - } - - ln := len(ids) - // todo: check what happens if there is more than maxKeys objects - if ln > p.MaxKeys { - ln = p.MaxKeys - } - - mostRecentModified := time.Time{} - needDirectoryAsKey := p.Version == 2 && len(p.Prefix) > 0 && len(p.Delimiter) > 0 && strings.HasSuffix(p.Prefix, p.Delimiter) - result.Objects = make([]*ObjectInfo, 0, ln) - - for _, id := range ids { - addr := object.NewAddress() - addr.SetObjectID(id) - addr.SetContainerID(bkt.CID) - - meta, err := n.objectHead(ctx, addr) - if err != nil { - n.log.Warn("could not fetch object meta", zap.Error(err)) - continue - } - - // // ignore tombstone objects - // _, hdr := meta.LastHeader(object.HeaderType(object.TombstoneHdr)) - // if hdr != nil { - // continue - // } - - // ignore storage group objects - // _, hdr = meta.LastHeader(object.HeaderType(object.StorageGroupHdr)) - // if hdr != nil { - // continue - // } - - // dirs don't exist in neofs, gateway stores full path to the file - // in object header, e.g. `filename`:`/this/is/path/file.txt` - - // prefix argument contains full dir path from the root, e.g. `/this/is/` - - // to emulate dirs we take dirs in path, compare it with prefix - // and look for entities after prefix. If entity does not have any - // sub-entities, then it is a file, else directory. - - if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { - if needDirectoryAsKey && oi.Created.After(mostRecentModified) { - mostRecentModified = oi.Created - } - // use only unique dir names - if _, ok := uniqNames[oi.Name]; ok { - continue - } - if len(p.Marker) > 0 && oi.Name <= p.Marker { - continue - } - - uniqNames[oi.Name] = oi.isDir - - result.Objects = append(result.Objects, oi) - } - } - - sort.Slice(result.Objects, func(i, j int) bool { - return result.Objects[i].Name < result.Objects[j].Name - }) - - if len(result.Objects) > p.MaxKeys { - result.IsTruncated = true - result.Objects = result.Objects[:p.MaxKeys] - result.NextMarker = result.Objects[len(result.Objects)-1].Name - } - - fillPrefixes(&result, uniqNames) - if needDirectoryAsKey { - res := []*ObjectInfo{{ - Name: p.Prefix, - Created: mostRecentModified, - HashSum: emptyETag, - }} - result.Objects = append(res, result.Objects...) - } - - return &result, nil -} - -func fillPrefixes(result *ListObjectsInfo, directories map[string]bool) { - index := 0 - for range result.Objects { - name := result.Objects[index].Name - if isDir := directories[name]; isDir { - result.Objects = append(result.Objects[:index], result.Objects[index+1:]...) - result.Prefixes = append(result.Prefixes, name) - } else { - index++ - } - } -} - // GetObject from storage. func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { var ( diff --git a/api/layer/layer_test.go b/api/layer/layer_test.go deleted file mode 100644 index c079e6f1..00000000 --- a/api/layer/layer_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package layer - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFillingPrefixes(t *testing.T) { - cases := []struct { - name string - list *ListObjectsInfo - directories map[string]bool - expectedPrefixes []string - expectedObjects []*ObjectInfo - }{ - { - name: "3 dirs", - list: &ListObjectsInfo{ - Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}}, - }, - directories: map[string]bool{"dir/": true, "dir2/": true, "dir3/": true}, - expectedPrefixes: []string{"dir/", "dir2/", "dir3/"}, - expectedObjects: []*ObjectInfo{}, - }, - { - name: "1 obj, 3 dirs", - list: &ListObjectsInfo{ - Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}, {Name: "obj"}}, - }, - directories: map[string]bool{"dir/": true, "dir2/": true, "dir3/": true}, - expectedPrefixes: []string{"dir/", "dir2/", "dir3/"}, - expectedObjects: []*ObjectInfo{{Name: "obj"}}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - fillPrefixes(tc.list, tc.directories) - require.Equal(t, tc.expectedPrefixes, tc.list.Prefixes) - require.Equal(t, tc.expectedObjects, tc.list.Objects) - }) - } -} diff --git a/api/layer/object.go b/api/layer/object.go index 4fe56f92..9ae56ceb 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/url" + "sort" "strconv" "time" @@ -12,6 +13,7 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-s3-gw/api" + "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -30,6 +32,35 @@ type ( length int64 address *object.Address } + + // ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2. + ListObjectsParamsCommon struct { + Bucket string + Delimiter string + Encode string + MaxKeys int + Prefix string + } + + // ListObjectsParamsV1 contains params for ListObjectsV1. + ListObjectsParamsV1 struct { + ListObjectsParamsCommon + Marker string + } + + // ListObjectsParamsV2 contains params for ListObjectsV2. + ListObjectsParamsV2 struct { + ListObjectsParamsCommon + ContinuationToken string + StartAfter string + } + + allObjectParams struct { + Bucket string + Delimiter string + Prefix string + StartAfter string + } ) // objectSearch returns all available objects by search params. @@ -170,3 +201,135 @@ func (n *layer) objectDelete(ctx context.Context, address *object.Address) error dop.WithAddress(address) return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx)) } + +// ListObjectsV1 returns objects in a bucket for requests of Version 1. +func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) { + var ( + err error + result ListObjectsInfoV1 + ) + + if p.MaxKeys == 0 { + return &result, nil + } + + allObjects, err := n.listSortedAllObjects(ctx, allObjectParams{ + Bucket: p.Bucket, + Prefix: p.Prefix, + Delimiter: p.Delimiter, + StartAfter: p.Marker, + }) + if err != nil { + return nil, err + } + + if len(allObjects) > p.MaxKeys { + result.IsTruncated = true + + nextObject := allObjects[p.MaxKeys-1] + result.NextMarker = nextObject.Name + + allObjects = allObjects[:p.MaxKeys] + } + + for _, ov := range allObjects { + if ov.isDir { + result.Prefixes = append(result.Prefixes, ov.Name) + } else { + result.Objects = append(result.Objects, ov) + } + } + + return &result, nil +} + +// ListObjectsV2 returns objects in a bucket for requests of Version 2. +func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) { + var ( + err error + result ListObjectsInfoV2 + allObjects []*ObjectInfo + ) + + if p.MaxKeys == 0 { + return &result, nil + } + + if p.ContinuationToken != "" { + // find cache with continuation token + } else { + allObjects, err = n.listSortedAllObjects(ctx, allObjectParams{ + Bucket: p.Bucket, + Prefix: p.Prefix, + Delimiter: p.Delimiter, + StartAfter: p.StartAfter, + }) + if err != nil { + return nil, err + } + } + + if len(allObjects) > p.MaxKeys { + result.IsTruncated = true + + allObjects = allObjects[:p.MaxKeys] + // add creating of cache here + } + + for _, ov := range allObjects { + if ov.isDir { + result.Prefixes = append(result.Prefixes, ov.Name) + } else { + result.Objects = append(result.Objects, ov) + } + } + return &result, nil +} + +func (n *layer) listSortedAllObjects(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { + var ( + err error + bkt *BucketInfo + ids []*object.ID + uniqNames = make(map[string]bool) + ) + + if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { + return nil, err + } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { + return nil, err + } + + objects := make([]*ObjectInfo, 0, len(ids)) + + for _, id := range ids { + addr := object.NewAddress() + addr.SetObjectID(id) + addr.SetContainerID(bkt.CID) + + meta, err := n.objectHead(ctx, addr) + if err != nil { + n.log.Warn("could not fetch object meta", zap.Error(err)) + continue + } + if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { + // use only unique dir names + if _, ok := uniqNames[oi.Name]; ok { + continue + } + if len(p.StartAfter) > 0 && oi.Name <= p.StartAfter { + continue + } + + uniqNames[oi.Name] = oi.isDir + + objects = append(objects, oi) + } + } + + sort.Slice(objects, func(i, j int) bool { + return objects[i].Name < objects[j].Name + }) + + return objects, nil +} diff --git a/api/layer/util.go b/api/layer/util.go index abd8dfd2..9f4247f1 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -26,32 +26,23 @@ type ( Headers map[string]string } - // ListObjectsInfo - container for list objects. + // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. ListObjectsInfo struct { - // Indicates whether the returned list objects response is truncated. A - // value of true indicates that the list was truncated. The list can be truncated - // if the number of objects exceeds the limit allowed or specified - // by max keys. + Prefixes []string + Objects []*ObjectInfo IsTruncated bool + } - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. - // - // NOTE: This element is returned only if you have delimiter request parameter - // specified. - ContinuationToken string - NextContinuationToken string - - // When response is truncated (the IsTruncated element value in the response is true), - // you can use the key name in this field as marker in the subsequent request to get next set of objects. + // ListObjectsInfoV1 holds data which ListObjectsV1 returns. + ListObjectsInfoV1 struct { + ListObjectsInfo NextMarker string + } - // List of objects info for this request. - Objects []*ObjectInfo - - // List of prefixes for this request. - Prefixes []string + // ListObjectsInfoV2 holds data which ListObjectsV2 returns. + ListObjectsInfoV2 struct { + ListObjectsInfo + NextContinuationToken string } // ObjectVersionInfo stores info about objects versions. From b1cda2a7147dcb0ee27f3c00c777b168fc35b030 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Mon, 19 Jul 2021 12:10:28 +0300 Subject: [PATCH 2/2] [#154] api: refactor EncodeV1 and EncodeV2 Move common parts of Encode to separate functions. Signed-off-by: Angira Kekteeva --- api/handler/object_list.go | 75 +++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/api/handler/object_list.go b/api/handler/object_list.go index 0e7bbb3a..63eb6999 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -42,28 +42,9 @@ func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *List NextMarker: list.NextMarker, } - // fill common prefixes - for i := range list.Prefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ - Prefix: s3PathEncode(list.Prefixes[i], p.Encode), - }) - } + res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode) - // fill contents - for _, obj := range list.Objects { - res.Contents = append(res.Contents, Object{ - Key: s3PathEncode(obj.Name, p.Encode), - Size: obj.Size, - LastModified: obj.Created.Format(time.RFC3339), - - Owner: Owner{ - ID: obj.Owner.String(), - DisplayName: obj.Owner.String(), - }, - - ETag: obj.HashSum, - }) - } + res.Contents = fillContents(list.Objects, p.Encode) return res } @@ -104,28 +85,9 @@ func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *List NextContinuationToken: list.NextContinuationToken, } - // fill common prefixes - for i := range list.Prefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ - Prefix: s3PathEncode(list.Prefixes[i], p.Encode), - }) - } + res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode) - // fill contents - for _, obj := range list.Objects { - res.Contents = append(res.Contents, Object{ - Key: s3PathEncode(obj.Name, p.Encode), - Size: obj.Size, - LastModified: obj.Created.Format(time.RFC3339), - - Owner: Owner{ - ID: obj.Owner.String(), - DisplayName: obj.Owner.String(), - }, - - ETag: obj.HashSum, - }) - } + res.Contents = fillContents(list.Objects, p.Encode) return res } @@ -188,6 +150,35 @@ func parseListObjectArgs(r *http.Request) (*layer.ListObjectsParamsCommon, error return &res, nil } +func fillPrefixes(src []string, encode string) []CommonPrefix { + var dst []CommonPrefix + for _, obj := range src { + dst = append(dst, CommonPrefix{ + Prefix: s3PathEncode(obj, encode), + }) + } + return dst +} + +func fillContents(src []*layer.ObjectInfo, encode string) []Object { + var dst []Object + for _, obj := range src { + dst = append(dst, Object{ + Key: s3PathEncode(obj.Name, encode), + Size: obj.Size, + LastModified: obj.Created.Format(time.RFC3339), + + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + + ETag: obj.HashSum, + }) + } + return dst +} + func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { p, err := parseListObjectVersionsRequest(r) if err != nil {