Merge pull request #111 from KirillovDenis/feature/97-listObject_paging

[#97] List object paging

api/errors.go

@@ -24,6 +24,7 @@ const (
 	ErrEntityTooSmall
 	ErrEntityTooLarge
 	ErrPolicyTooLarge
+	ErrIllegalVersioningConfigurationException
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
@@ -383,6 +384,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Policy exceeds the maximum allowed document size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrIllegalVersioningConfigurationException: {
+		Code:           "IllegalVersioningConfigurationException",
+		Description:    "Indicates that the versioning configuration specified in the request is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrIncompleteBody: {
 		Code:           "IncompleteBody",
 		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",

api/handler/list.go

@@ -17,9 +17,10 @@ type listObjectsArgs struct {
 	Delimeter  string
 	Encode     string
 	Marker     string
+	StartAfter string
 	MaxKeys    int
 	Prefix     string
-	Version   string
+	APIVersion int
 }
 
 // VersioningConfiguration contains VersioningConfiguration XML representation.
@@ -114,11 +115,17 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
 		return nil, nil, err
 	}
 
+	marker := arg.Marker
+	if arg.APIVersion == 2 {
+		marker = arg.StartAfter
+	}
+
 	list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
 		Bucket:    arg.Bucket,
 		Prefix:    arg.Prefix,
 		MaxKeys:   arg.MaxKeys,
 		Delimiter: arg.Delimeter,
+		Marker:    marker,
 	})
 	if err != nil {
 		h.log.Error("something went wrong",
@@ -166,7 +173,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
 		Delimiter:    arg.Delimeter,
 
 		IsTruncated: list.IsTruncated,
-		NextMarker:  list.NextContinuationToken,
+		NextMarker:  list.NextMarker,
 	}
 
 	// fill common prefixes
@@ -221,8 +228,10 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
 		Name:         arg.Bucket,
 		EncodingType: arg.Encode,
 		Prefix:       arg.Prefix,
+		KeyCount:     len(list.Objects),
 		MaxKeys:      arg.MaxKeys,
 		Delimiter:    arg.Delimeter,
+		StartAfter:   arg.StartAfter,
 
 		IsTruncated: list.IsTruncated,
 
@@ -271,10 +280,19 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
 	}
 
 	res.Prefix = r.URL.Query().Get("prefix")
-	res.Marker = r.URL.Query().Get("key-marker")
+	res.Marker = r.URL.Query().Get("marker")
 	res.Delimeter = r.URL.Query().Get("delimiter")
 	res.Encode = r.URL.Query().Get("encoding-type")
-	res.Version = r.URL.Query().Get("version-id-marker")
+	res.StartAfter = r.URL.Query().Get("start-after")
+	apiVersionStr := r.URL.Query().Get("list-type")
+
+	res.APIVersion = 1
+	if len(apiVersionStr) != 0 {
+		if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 {
+			return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)
+		}
+		res.APIVersion = 2
+	}
 
 	if info := api.GetReqInfo(r.Context()); info != nil {
 		res.Bucket = info.BucketName

api/layer/container.go

@@ -31,6 +31,7 @@ type (
 		Token     string
 		Delimiter string
 		MaxKeys   int
+		Marker    string
 	}
 )
 

api/layer/layer.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"sort"
 	"time"
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/client"
@@ -213,7 +214,6 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 	// todo: check what happens if there is more than maxKeys objects
 	if ln > p.MaxKeys {
 		ln = p.MaxKeys
-		result.IsTruncated = true
 	}
 
 	result.Objects = make([]*ObjectInfo, 0, ln)
@@ -253,6 +253,9 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
 			// use only unique dir names
 			if _, ok := uniqNames[oi.Name]; !ok {
+				if len(p.Marker) > 0 && oi.Name <= p.Marker {
+					continue
+				}
 				uniqNames[oi.Name] = struct{}{}
 
 				result.Objects = append(result.Objects, oi)
@@ -260,6 +263,15 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		}
 	}
 
+	sort.Slice(result.Objects, func(i, j int) bool {
+		return result.Objects[i].Name < result.Objects[j].Name
+	})
+
+	if len(result.Objects) > p.MaxKeys {
+		result.IsTruncated = true
+		result.Objects = result.Objects[:p.MaxKeys]
+		result.NextMarker = result.Objects[len(result.Objects)-1].Name
+	}
 	return &result, nil
 }
 

api/layer/util.go

@@ -43,6 +43,10 @@ type (
 		ContinuationToken     string
 		NextContinuationToken string
 
+		// When response is truncated (the IsTruncated element value in the response is true),
+		// you can use the key name in this field as marker in the subsequent request to get next set of objects.
+		NextMarker string
+
 		// List of objects info for this request.
 		Objects []*ObjectInfo