[#97] List object paging

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-06-25 15:54:25 +03:00
parent ce65a47d1b
commit 23dfa0c46e
5 changed files with 52 additions and 11 deletions

View file

@ -24,6 +24,7 @@ const (
ErrEntityTooSmall ErrEntityTooSmall
ErrEntityTooLarge ErrEntityTooLarge
ErrPolicyTooLarge ErrPolicyTooLarge
ErrIllegalVersioningConfigurationException
ErrIncompleteBody ErrIncompleteBody
ErrInternalError ErrInternalError
ErrInvalidAccessKeyID ErrInvalidAccessKeyID
@ -383,6 +384,11 @@ var errorCodes = errorCodeMap{
Description: "Policy exceeds the maximum allowed document size.", Description: "Policy exceeds the maximum allowed document size.",
HTTPStatusCode: http.StatusBadRequest, HTTPStatusCode: http.StatusBadRequest,
}, },
ErrIllegalVersioningConfigurationException: {
Code: "IllegalVersioningConfigurationException",
Description: "Indicates that the versioning configuration specified in the request is invalid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrIncompleteBody: { ErrIncompleteBody: {
Code: "IncompleteBody", Code: "IncompleteBody",
Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.", Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.",

View file

@ -13,13 +13,14 @@ import (
) )
type listObjectsArgs struct { type listObjectsArgs struct {
Bucket string Bucket string
Delimeter string Delimeter string
Encode string Encode string
Marker string Marker string
MaxKeys int StartAfter string
Prefix string MaxKeys int
Version string Prefix string
APIVersion int
} }
// VersioningConfiguration contains VersioningConfiguration XML representation. // VersioningConfiguration contains VersioningConfiguration XML representation.
@ -114,11 +115,17 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
return nil, nil, err return nil, nil, err
} }
marker := arg.Marker
if arg.APIVersion == 2 {
marker = arg.StartAfter
}
list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{ list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
Bucket: arg.Bucket, Bucket: arg.Bucket,
Prefix: arg.Prefix, Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter, Delimiter: arg.Delimeter,
Marker: marker,
}) })
if err != nil { if err != nil {
h.log.Error("something went wrong", h.log.Error("something went wrong",
@ -166,7 +173,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
Delimiter: arg.Delimeter, Delimiter: arg.Delimeter,
IsTruncated: list.IsTruncated, IsTruncated: list.IsTruncated,
NextMarker: list.NextContinuationToken, NextMarker: list.NextMarker,
} }
// fill common prefixes // fill common prefixes
@ -221,8 +228,10 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
Name: arg.Bucket, Name: arg.Bucket,
EncodingType: arg.Encode, EncodingType: arg.Encode,
Prefix: arg.Prefix, Prefix: arg.Prefix,
KeyCount: len(list.Objects),
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter, Delimiter: arg.Delimeter,
StartAfter: arg.StartAfter,
IsTruncated: list.IsTruncated, IsTruncated: list.IsTruncated,
@ -271,10 +280,19 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
} }
res.Prefix = r.URL.Query().Get("prefix") res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("key-marker") res.Marker = r.URL.Query().Get("marker")
res.Delimeter = r.URL.Query().Get("delimiter") res.Delimeter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type") res.Encode = r.URL.Query().Get("encoding-type")
res.Version = r.URL.Query().Get("version-id-marker") res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type")
res.APIVersion = 1
if len(apiVersionStr) != 0 {
if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 {
return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)
}
res.APIVersion = 2
}
if info := api.GetReqInfo(r.Context()); info != nil { if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName res.Bucket = info.BucketName

View file

@ -31,6 +31,7 @@ type (
Token string Token string
Delimiter string Delimiter string
MaxKeys int MaxKeys int
Marker string
} }
) )

View file

@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"sort"
"time" "time"
"github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/client"
@ -213,7 +214,6 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
// todo: check what happens if there is more than maxKeys objects // todo: check what happens if there is more than maxKeys objects
if ln > p.MaxKeys { if ln > p.MaxKeys {
ln = p.MaxKeys ln = p.MaxKeys
result.IsTruncated = true
} }
result.Objects = make([]*ObjectInfo, 0, ln) result.Objects = make([]*ObjectInfo, 0, ln)
@ -253,6 +253,9 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil { if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
// use only unique dir names // use only unique dir names
if _, ok := uniqNames[oi.Name]; !ok { if _, ok := uniqNames[oi.Name]; !ok {
if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue
}
uniqNames[oi.Name] = struct{}{} uniqNames[oi.Name] = struct{}{}
result.Objects = append(result.Objects, oi) result.Objects = append(result.Objects, oi)
@ -260,6 +263,15 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
} }
} }
sort.Slice(result.Objects, func(i, j int) bool {
return result.Objects[i].Name < result.Objects[j].Name
})
if len(result.Objects) > p.MaxKeys {
result.IsTruncated = true
result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name
}
return &result, nil return &result, nil
} }

View file

@ -43,6 +43,10 @@ type (
ContinuationToken string ContinuationToken string
NextContinuationToken string NextContinuationToken string
// When response is truncated (the IsTruncated element value in the response is true),
// you can use the key name in this field as marker in the subsequent request to get next set of objects.
NextMarker string
// List of objects info for this request. // List of objects info for this request.
Objects []*ObjectInfo Objects []*ObjectInfo