[#97] List object paging

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-06-25 15:54:25 +03:00
parent ce65a47d1b
commit 23dfa0c46e
5 changed files with 52 additions and 11 deletions

View file

@ -24,6 +24,7 @@ const (
ErrEntityTooSmall
ErrEntityTooLarge
ErrPolicyTooLarge
ErrIllegalVersioningConfigurationException
ErrIncompleteBody
ErrInternalError
ErrInvalidAccessKeyID
@ -383,6 +384,11 @@ var errorCodes = errorCodeMap{
Description: "Policy exceeds the maximum allowed document size.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrIllegalVersioningConfigurationException: {
Code: "IllegalVersioningConfigurationException",
Description: "Indicates that the versioning configuration specified in the request is invalid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrIncompleteBody: {
Code: "IncompleteBody",
Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.",

View file

@ -13,13 +13,14 @@ import (
)
type listObjectsArgs struct {
Bucket string
Delimeter string
Encode string
Marker string
MaxKeys int
Prefix string
Version string
Bucket string
Delimeter string
Encode string
Marker string
StartAfter string
MaxKeys int
Prefix string
APIVersion int
}
// VersioningConfiguration contains VersioningConfiguration XML representation.
@ -114,11 +115,17 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
return nil, nil, err
}
marker := arg.Marker
if arg.APIVersion == 2 {
marker = arg.StartAfter
}
list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
Bucket: arg.Bucket,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
Marker: marker,
})
if err != nil {
h.log.Error("something went wrong",
@ -166,7 +173,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
Delimiter: arg.Delimeter,
IsTruncated: list.IsTruncated,
NextMarker: list.NextContinuationToken,
NextMarker: list.NextMarker,
}
// fill common prefixes
@ -221,8 +228,10 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
Name: arg.Bucket,
EncodingType: arg.Encode,
Prefix: arg.Prefix,
KeyCount: len(list.Objects),
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
StartAfter: arg.StartAfter,
IsTruncated: list.IsTruncated,
@ -271,10 +280,19 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
}
res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("key-marker")
res.Marker = r.URL.Query().Get("marker")
res.Delimeter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.Version = r.URL.Query().Get("version-id-marker")
res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type")
res.APIVersion = 1
if len(apiVersionStr) != 0 {
if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 {
return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)
}
res.APIVersion = 2
}
if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName

View file

@ -31,6 +31,7 @@ type (
Token string
Delimiter string
MaxKeys int
Marker string
}
)

View file

@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/url"
"sort"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
@ -213,7 +214,6 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
// todo: check what happens if there is more than maxKeys objects
if ln > p.MaxKeys {
ln = p.MaxKeys
result.IsTruncated = true
}
result.Objects = make([]*ObjectInfo, 0, ln)
@ -253,6 +253,9 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
// use only unique dir names
if _, ok := uniqNames[oi.Name]; !ok {
if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue
}
uniqNames[oi.Name] = struct{}{}
result.Objects = append(result.Objects, oi)
@ -260,6 +263,15 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
}
}
sort.Slice(result.Objects, func(i, j int) bool {
return result.Objects[i].Name < result.Objects[j].Name
})
if len(result.Objects) > p.MaxKeys {
result.IsTruncated = true
result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name
}
return &result, nil
}

View file

@ -43,6 +43,10 @@ type (
ContinuationToken string
NextContinuationToken string
// When response is truncated (the IsTruncated element value in the response is true),
// you can use the key name in this field as marker in the subsequent request to get next set of objects.
NextMarker string
// List of objects info for this request.
Objects []*ObjectInfo