From b2e8b1cfb320efedc842d638f93afb19abe3880d Mon Sep 17 00:00:00 2001
From: Denis Kirillov <denis@nspcc.ru>
Date: Mon, 25 Jul 2022 16:46:36 +0300
Subject: [PATCH] [#612] Make listing more robust

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
---
 api/handler/object_list_test.go | 47 +++++++++++++++++++++++++++++++++
 api/layer/object.go             | 12 +++++++--
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/api/handler/object_list_test.go b/api/handler/object_list_test.go
index d28175a8..33ba4528 100644
--- a/api/handler/object_list_test.go
+++ b/api/handler/object_list_test.go
@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"net/http"
+	"net/url"
+	"strconv"
 	"testing"
 
 	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
@@ -73,3 +75,48 @@ func TestListObjectNullVersions(t *testing.T) {
 	require.Len(t, result.Version, 2)
 	require.Equal(t, layer.UnversionedObjectVersionID, result.Version[1].VersionID)
 }
+
+func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) {
+	tc := prepareHandlerContext(t)
+
+	bktName := "bucket-for-listing"
+	objects := []string{"bar", "baz", "foo", "quxx"}
+	bktInfo, _ := createBucketAndObject(t, tc, bktName, objects[0])
+
+	for _, objName := range objects[1:] {
+		createTestObject(tc.Context(), t, tc, bktInfo, objName)
+	}
+
+	listV2Response1 := listObjectsV2(t, tc, bktName, "bar", "", 1)
+	nextContinuationToken := listV2Response1.NextContinuationToken
+	require.Equal(t, "baz", listV2Response1.Contents[0].Key)
+
+	listV2Response2 := listObjectsV2(t, tc, bktName, "bar", nextContinuationToken, -1)
+
+	require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
+	require.Equal(t, "bar", listV2Response2.StartAfter)
+	require.False(t, listV2Response2.IsTruncated)
+
+	require.Equal(t, "foo", listV2Response2.Contents[0].Key)
+	require.Equal(t, "quxx", listV2Response2.Contents[1].Key)
+}
+
+func listObjectsV2(t *testing.T, tc *handlerContext, bktName, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
+	query := make(url.Values)
+	if len(startAfter) != 0 {
+		query.Add("start-after", startAfter)
+	}
+	if len(continuationToken) != 0 {
+		query.Add("continuation-token", continuationToken)
+	}
+	if maxKeys != -1 {
+		query.Add("max-keys", strconv.Itoa(maxKeys))
+	}
+
+	w, r := prepareTestFullRequest(t, bktName, "", query, nil)
+	tc.Handler().ListObjectsV2Handler(w, r)
+	assertStatus(t, w, http.StatusOK)
+	res := &ListObjectsV2Response{}
+	parseTestResponse(t, w, res)
+	return res
+}
diff --git a/api/layer/object.go b/api/layer/object.go
index 7ff32a3e..8bd7727a 100644
--- a/api/layer/object.go
+++ b/api/layer/object.go
@@ -471,8 +471,11 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
 	objects = make([]*data.ObjectInfo, 0, p.MaxKeys)
 
 	for obj := range objOutCh {
-		if len(objects) == p.MaxKeys { // todo reconsider stop condition
-			next = obj
+		// TODO (@kirillovdenis) : #612, #525 reconsider stop condition
+		// currently we handle 3 more items to reduce the likelihood of missing the last object in batch
+		// (potentially we can miss it because of pool of workers)
+		if len(objects) == p.MaxKeys+3 {
+			//next = obj
 			break
 		}
 		objects = append(objects, obj)
@@ -482,6 +485,11 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
 		return objects[i].Name < objects[j].Name
 	})
 
+	if len(objects) > p.MaxKeys {
+		next = objects[p.MaxKeys]
+		objects = objects[:p.MaxKeys]
+	}
+
 	return
 }