[#612] Make listing more robust

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-07-25 16:46:36 +03:00 committed by Alex Vanin
parent 4483c6f57a
commit b2e8b1cfb3
2 changed files with 57 additions and 2 deletions

View file

@ -4,6 +4,8 @@ import (
"bytes" "bytes"
"context" "context"
"net/http" "net/http"
"net/url"
"strconv"
"testing" "testing"
"github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/layer"
@ -73,3 +75,48 @@ func TestListObjectNullVersions(t *testing.T) {
require.Len(t, result.Version, 2) require.Len(t, result.Version, 2)
require.Equal(t, layer.UnversionedObjectVersionID, result.Version[1].VersionID) require.Equal(t, layer.UnversionedObjectVersionID, result.Version[1].VersionID)
} }
func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) {
tc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
objects := []string{"bar", "baz", "foo", "quxx"}
bktInfo, _ := createBucketAndObject(t, tc, bktName, objects[0])
for _, objName := range objects[1:] {
createTestObject(tc.Context(), t, tc, bktInfo, objName)
}
listV2Response1 := listObjectsV2(t, tc, bktName, "bar", "", 1)
nextContinuationToken := listV2Response1.NextContinuationToken
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
listV2Response2 := listObjectsV2(t, tc, bktName, "bar", nextContinuationToken, -1)
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
require.Equal(t, "bar", listV2Response2.StartAfter)
require.False(t, listV2Response2.IsTruncated)
require.Equal(t, "foo", listV2Response2.Contents[0].Key)
require.Equal(t, "quxx", listV2Response2.Contents[1].Key)
}
func listObjectsV2(t *testing.T, tc *handlerContext, bktName, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
query := make(url.Values)
if len(startAfter) != 0 {
query.Add("start-after", startAfter)
}
if len(continuationToken) != 0 {
query.Add("continuation-token", continuationToken)
}
if maxKeys != -1 {
query.Add("max-keys", strconv.Itoa(maxKeys))
}
w, r := prepareTestFullRequest(t, bktName, "", query, nil)
tc.Handler().ListObjectsV2Handler(w, r)
assertStatus(t, w, http.StatusOK)
res := &ListObjectsV2Response{}
parseTestResponse(t, w, res)
return res
}

View file

@ -471,8 +471,11 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
objects = make([]*data.ObjectInfo, 0, p.MaxKeys) objects = make([]*data.ObjectInfo, 0, p.MaxKeys)
for obj := range objOutCh { for obj := range objOutCh {
if len(objects) == p.MaxKeys { // todo reconsider stop condition // TODO (@kirillovdenis) : #612, #525 reconsider stop condition
next = obj // currently we handle 3 more items to reduce the likelihood of missing the last object in batch
// (potentially we can miss it because of pool of workers)
if len(objects) == p.MaxKeys+3 {
//next = obj
break break
} }
objects = append(objects, obj) objects = append(objects, obj)
@ -482,6 +485,11 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
return objects[i].Name < objects[j].Name return objects[i].Name < objects[j].Name
}) })
if len(objects) > p.MaxKeys {
next = objects[p.MaxKeys]
objects = objects[:p.MaxKeys]
}
return return
} }