From 23dfa0c46e1c12227b17ccc4bbca567f9fa8b42e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 25 Jun 2021 15:54:25 +0300 Subject: [PATCH] [#97] List object paging Signed-off-by: Denis Kirillov --- api/errors.go | 6 ++++++ api/handler/list.go | 38 ++++++++++++++++++++++++++++---------- api/layer/container.go | 1 + api/layer/layer.go | 14 +++++++++++++- api/layer/util.go | 4 ++++ 5 files changed, 52 insertions(+), 11 deletions(-) diff --git a/api/errors.go b/api/errors.go index 6fb52b4e..a619193f 100644 --- a/api/errors.go +++ b/api/errors.go @@ -24,6 +24,7 @@ const ( ErrEntityTooSmall ErrEntityTooLarge ErrPolicyTooLarge + ErrIllegalVersioningConfigurationException ErrIncompleteBody ErrInternalError ErrInvalidAccessKeyID @@ -383,6 +384,11 @@ var errorCodes = errorCodeMap{ Description: "Policy exceeds the maximum allowed document size.", HTTPStatusCode: http.StatusBadRequest, }, + ErrIllegalVersioningConfigurationException: { + Code: "IllegalVersioningConfigurationException", + Description: "Indicates that the versioning configuration specified in the request is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, ErrIncompleteBody: { Code: "IncompleteBody", Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.", diff --git a/api/handler/list.go b/api/handler/list.go index a55d68ce..35e74a79 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -13,13 +13,14 @@ import ( ) type listObjectsArgs struct { - Bucket string - Delimeter string - Encode string - Marker string - MaxKeys int - Prefix string - Version string + Bucket string + Delimeter string + Encode string + Marker string + StartAfter string + MaxKeys int + Prefix string + APIVersion int } // VersioningConfiguration contains VersioningConfiguration XML representation. @@ -114,11 +115,17 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje return nil, nil, err } + marker := arg.Marker + if arg.APIVersion == 2 { + marker = arg.StartAfter + } + list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{ Bucket: arg.Bucket, Prefix: arg.Prefix, MaxKeys: arg.MaxKeys, Delimiter: arg.Delimeter, + Marker: marker, }) if err != nil { h.log.Error("something went wrong", @@ -166,7 +173,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes Delimiter: arg.Delimeter, IsTruncated: list.IsTruncated, - NextMarker: list.NextContinuationToken, + NextMarker: list.NextMarker, } // fill common prefixes @@ -221,8 +228,10 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R Name: arg.Bucket, EncodingType: arg.Encode, Prefix: arg.Prefix, + KeyCount: len(list.Objects), MaxKeys: arg.MaxKeys, Delimiter: arg.Delimeter, + StartAfter: arg.StartAfter, IsTruncated: list.IsTruncated, @@ -271,10 +280,19 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { } res.Prefix = r.URL.Query().Get("prefix") - res.Marker = r.URL.Query().Get("key-marker") + res.Marker = r.URL.Query().Get("marker") res.Delimeter = r.URL.Query().Get("delimiter") res.Encode = r.URL.Query().Get("encoding-type") - res.Version = r.URL.Query().Get("version-id-marker") + res.StartAfter = r.URL.Query().Get("start-after") + apiVersionStr := r.URL.Query().Get("list-type") + + res.APIVersion = 1 + if len(apiVersionStr) != 0 { + if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 { + return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException) + } + res.APIVersion = 2 + } if info := api.GetReqInfo(r.Context()); info != nil { res.Bucket = info.BucketName diff --git a/api/layer/container.go b/api/layer/container.go index eec019c4..130ca583 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -31,6 +31,7 @@ type ( Token string Delimiter string MaxKeys int + Marker string } ) diff --git a/api/layer/layer.go b/api/layer/layer.go index a7c7c3ba..a81ba923 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/url" + "sort" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -213,7 +214,6 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj // todo: check what happens if there is more than maxKeys objects if ln > p.MaxKeys { ln = p.MaxKeys - result.IsTruncated = true } result.Objects = make([]*ObjectInfo, 0, ln) @@ -253,6 +253,9 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil { // use only unique dir names if _, ok := uniqNames[oi.Name]; !ok { + if len(p.Marker) > 0 && oi.Name <= p.Marker { + continue + } uniqNames[oi.Name] = struct{}{} result.Objects = append(result.Objects, oi) @@ -260,6 +263,15 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj } } + sort.Slice(result.Objects, func(i, j int) bool { + return result.Objects[i].Name < result.Objects[j].Name + }) + + if len(result.Objects) > p.MaxKeys { + result.IsTruncated = true + result.Objects = result.Objects[:p.MaxKeys] + result.NextMarker = result.Objects[len(result.Objects)-1].Name + } return &result, nil } diff --git a/api/layer/util.go b/api/layer/util.go index 96656095..ba60cc7c 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -43,6 +43,10 @@ type ( ContinuationToken string NextContinuationToken string + // When response is truncated (the IsTruncated element value in the response is true), + // you can use the key name in this field as marker in the subsequent request to get next set of objects. + NextMarker string + // List of objects info for this request. Objects []*ObjectInfo