frostfs-s3-gw/api/handler/object_list_test.go

package handler

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"

	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
	"github.com/stretchr/testify/require"
)

func TestParseContinuationToken(t *testing.T) {
	var err error

	t.Run("empty token", func(t *testing.T) {
		var queryValues = map[string][]string{
			"continuation-token": {""},
		}
		_, err = parseContinuationToken(queryValues)
		require.Error(t, err)
	})

	t.Run("invalid not empty token", func(t *testing.T) {
		var queryValues = map[string][]string{
			"continuation-token": {"asd"},
		}
		_, err = parseContinuationToken(queryValues)
		require.Error(t, err)
	})

	t.Run("valid token", func(t *testing.T) {
		tokenStr := "75BTT5Z9o79XuKdUeGqvQbqDnxu6qWcR5EhxW8BXFf8t"
		var queryValues = map[string][]string{
			"continuation-token": {tokenStr},
		}
		token, err := parseContinuationToken(queryValues)
		require.NoError(t, err)
		require.Equal(t, tokenStr, token)
	})
}

func TestListObjectNullVersions(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName, objName := "bucket-versioning-enabled", "object"
	createTestBucket(hc, bktName)

	putObjectContent(hc, bktName, objName, "content")
	putBucketVersioning(t, hc, bktName, true)
	putObjectContent(hc, bktName, objName, "content2")

	result := listVersions(t, hc, bktName)

	require.Len(t, result.Version, 2)
	require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID)
}

func TestListObjectsPaging(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-versioning-enabled"
	createTestBucket(hc, bktName)

	n := 10

	var objects []string
	for i := 0; i < n; i++ {
		objects = append(objects, "objects"+strconv.Itoa(i))
		putObjectContent(hc, bktName, objects[i], "content")
	}
	sort.Strings(objects)

	result := &ListObjectsVersionsResponse{IsTruncated: true}
	for result.IsTruncated {
		result = listObjectsVersions(hc, bktName, "", "", result.NextKeyMarker, result.NextVersionIDMarker, n/3)

		for i, version := range result.Version {
			if objects[i] != version.Key {
				t.Errorf("expected: '%s', got: '%s'", objects[i], version.Key)
			}
		}
		objects = objects[len(result.Version):]
	}

	require.Empty(t, objects)
}

func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) {
	tc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	objects := []string{"bar", "baz", "foo", "quxx"}
	bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])

	for _, objName := range objects[1:] {
		createTestObject(tc, bktInfo, objName, encryption.Params{})
	}

	listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
	nextContinuationToken := listV2Response1.NextContinuationToken
	require.Equal(t, "baz", listV2Response1.Contents[0].Key)

	listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)

	require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
	require.Equal(t, "bar", listV2Response2.StartAfter)
	require.False(t, listV2Response2.IsTruncated)

	require.Equal(t, "foo", listV2Response2.Contents[0].Key)
	require.Equal(t, "quxx", listV2Response2.Contents[1].Key)
}

func TestS3BucketListV2EncodingBasic(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing-v1-encoding"
	bktInfo := createTestBucket(hc, bktName)

	objects := []string{"foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b"}
	for _, objName := range objects {
		createTestObject(hc, bktInfo, objName, encryption.Params{})
	}

	query := make(url.Values)
	query.Add("delimiter", "/")
	query.Add("encoding-type", "url")

	w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
	hc.Handler().ListObjectsV2Handler(w, r)
	assertStatus(hc.t, w, http.StatusOK)
	listV2Response := &ListObjectsV2Response{}
	parseTestResponse(hc.t, w, listV2Response)

	require.Equal(t, "/", listV2Response.Delimiter)
	require.Len(t, listV2Response.Contents, 1)
	require.Equal(t, "asdf%2Bb", listV2Response.Contents[0].Key)
	require.Len(t, listV2Response.CommonPrefixes, 3)
	require.Equal(t, "foo%2B1/", listV2Response.CommonPrefixes[0].Prefix)
	require.Equal(t, "foo/", listV2Response.CommonPrefixes[1].Prefix)
	require.Equal(t, "quux%20ab/", listV2Response.CommonPrefixes[2].Prefix)
}

func TestS3BucketListDelimiterBasic(t *testing.T) {
	tc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	objects := []string{"foo/bar", "foo/bar/xyzzy", "quux/thud", "asdf"}
	bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])

	for _, objName := range objects[1:] {
		createTestObject(tc, bktInfo, objName, encryption.Params{})
	}

	listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
	require.Equal(t, "/", listV1Response.Delimiter)
	require.Equal(t, "asdf", listV1Response.Contents[0].Key)
	require.Len(t, listV1Response.CommonPrefixes, 2)
	require.Equal(t, "foo/", listV1Response.CommonPrefixes[0].Prefix)
	require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
}

func TestS3BucketListV2PrefixAlt(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"bar", "baz", "foo"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "ba", "", "", "", -1)

	require.Equal(t, "ba", response.Prefix)
	require.Len(t, response.Contents, 2)
	require.Equal(t, "bar", response.Contents[0].Key)
	require.Equal(t, "baz", response.Contents[1].Key)
	require.Empty(t, response.CommonPrefixes)
}

func TestS3BucketListV2PrefixNotExist(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"foo/bar", "foo/baz", "quux"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "d", "", "", "", -1)

	require.Equal(t, "d", response.Prefix)
	require.Empty(t, response.Contents)
	require.Empty(t, response.CommonPrefixes)
}

func TestS3BucketListV2PrefixUnreadable(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"foo/bar", "foo/baz", "quux"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "\x0a", "", "", "", -1)

	require.Equal(t, "\x0a", response.Prefix)
	require.Empty(t, response.Contents)
	require.Empty(t, response.CommonPrefixes)
}

func TestS3BucketListV2PrefixDelimiterAlt(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"bar", "bazar", "cab", "foo"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "ba", "a", "", "", -1)

	require.Equal(t, "ba", response.Prefix)
	require.Equal(t, "a", response.Delimiter)
	require.Len(t, response.Contents, 1)
	require.Equal(t, "bar", response.Contents[0].Key)
	require.Len(t, response.CommonPrefixes, 1)
	require.Equal(t, "baza", response.CommonPrefixes[0].Prefix)
}

func TestS3BucketListV2PrefixDelimiterDelimiterNotExist(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "b", "z", "", "", -1)

	require.Len(t, response.Contents, 3)
	require.Equal(t, "b/a/c", response.Contents[0].Key)
	require.Equal(t, "b/a/g", response.Contents[1].Key)
	require.Equal(t, "b/a/r", response.Contents[2].Key)
	require.Empty(t, response.CommonPrefixes)
}

func TestS3BucketListV2PrefixDelimiterPrefixDelimiterNotExist(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	createTestBucket(hc, bktName)

	objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
	for _, objName := range objects {
		putObject(hc, bktName, objName)
	}

	response := listObjectsV2(hc, bktName, "y", "z", "", "", -1)

	require.Empty(t, response.Contents)
	require.Empty(t, response.CommonPrefixes)
}

func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
	tc := prepareHandlerContext(t)

	bktName := "bucket-for-listing"
	objects := []string{"b%ar", "b%az", "c%ab", "foo"}
	bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])

	for _, objName := range objects[1:] {
		createTestObject(tc, bktInfo, objName, encryption.Params{})
	}

	listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
	require.Equal(t, "%", listV2Response.Delimiter)
	require.Len(t, listV2Response.Contents, 1)
	require.Equal(t, "foo", listV2Response.Contents[0].Key)
	require.Len(t, listV2Response.CommonPrefixes, 2)
	require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
	require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
}

func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
	tc := prepareHandlerContext(t)

	bktName := "bucket-for-listingv2"
	objects := []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}
	bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])

	for _, objName := range objects[1:] {
		createTestObject(tc, bktInfo, objName, encryption.Params{})
	}

	var empty []string
	delim := "/"
	prefix := ""

	continuationToken := validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"asdf"}, empty)
	continuationToken = validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, true, false, empty, []string{"boo/"})
	validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"cquux/"})

	continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 2, true, false, []string{"asdf"}, []string{"boo/"})
	validateListV2(t, tc, bktName, prefix, delim, continuationToken, 2, false, true, empty, []string{"cquux/"})

	prefix = "boo/"
	continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"boo/bar"}, empty)
	validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"boo/baz/"})

	validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
}

func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName, objName := "mint-bucket-for-listing-versions", "objName"
	createTestBucket(hc, bktName)
	putBucketVersioning(t, hc, bktName, true)

	length := 10
	objects := make([]string, length)
	for i := 0; i < length; i++ {
		objects[i] = objName
		putObject(hc, bktName, objName)
	}

	maxKeys := 5

	page1 := listObjectsVersions(hc, bktName, "", "", "", "", maxKeys)
	require.Len(t, page1.Version, maxKeys)
	checkVersionsNames(t, page1, objects)
	require.Equal(t, page1.Version[maxKeys-1].VersionID, page1.NextVersionIDMarker)
	require.True(t, page1.IsTruncated)
	require.Empty(t, page1.KeyMarker)
	require.Empty(t, page1.VersionIDMarker)

	page2 := listObjectsVersions(hc, bktName, "", "", page1.NextKeyMarker, page1.NextVersionIDMarker, maxKeys)
	require.Len(t, page2.Version, maxKeys)
	checkVersionsNames(t, page1, objects)
	require.Empty(t, page2.NextVersionIDMarker)
	require.False(t, page2.IsTruncated)
	require.Equal(t, page1.NextKeyMarker, page2.KeyMarker)
	require.Equal(t, page1.NextVersionIDMarker, page2.VersionIDMarker)
}

func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
	for i, v := range versions.Version {
		require.Equal(t, names[i], v.Key)
	}
}

func TestHugeListV2(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listingv2"
	bktInfo := createTestBucket(hc, bktName)

	objects := prepareObjects(hc, bktInfo, "", 50005)

	fmt.Println("listing start")
	start := time.Now()

	resp := &ListObjectsV2Response{IsTruncated: true}
	for resp.IsTruncated {
		resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1)
		for i, content := range resp.Contents {
			if content.Key != objects[i] {
				t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
			}
		}
		objects = objects[len(resp.Contents):]
	}
	require.Empty(t, objects)

	fmt.Println(time.Since(start))
}

func TestListV2StreamNested1(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listingv2-nested"
	bktInfo := createTestBucket(hc, bktName)

	objects1 := prepareObjects(hc, bktInfo, "prefix", 10)
	objects2 := prepareObjects(hc, bktInfo, "prefix2", 10)

	objects := append(objects1, objects2...)

	fmt.Println("listing start")
	start := time.Now()

	resp := &ListObjectsV2Response{IsTruncated: true}
	for resp.IsTruncated {
		resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1)
		for i, content := range resp.Contents {
			if content.Key != objects[i] {
				t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
			}
		}
		objects = objects[len(resp.Contents):]
	}
	require.Empty(t, objects)

	fmt.Println(time.Since(start))
}

func TestHugeListV1(t *testing.T) {
	hc := prepareHandlerContext(t)

	bktName := "bucket-for-listingv1"
	bktInfo := createTestBucket(hc, bktName)

	objects := prepareObjects(hc, bktInfo, "", 50005)

	fmt.Println("listing start")
	start := time.Now()

	resp := &ListObjectsV1Response{IsTruncated: true}
	for resp.IsTruncated {
		resp = listObjectsV1(hc, bktName, "", "", resp.NextMarker, -1)
		for i, content := range resp.Contents {
			if content.Key != objects[i] {
				t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
			}
		}
		objects = objects[len(resp.Contents):]
	}

	require.Empty(t, objects)

	fmt.Println(time.Since(start))
}

func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string {
	treeID := "version"
	parentID := uint64(0)
	if prefix != "" {
		for _, filename := range strings.Split(prefix, "/") {
			nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{
				"FileName": filename,
			})
			require.NoError(hc.t, err)
			parentID = nodeID
		}
		prefix += "/"
	}

	objects := make([]string, size)

	for i := range objects {
		filename := "object" + strconv.Itoa(i)
		filepath := prefix + filename

		prm := layer.PrmObjectCreate{
			Container: bktInfo.CID,
			Filepath:  filepath,
			Payload:   nil,
		}

		id, err := hc.tp.CreateObject(hc.Context(), prm)
		require.NoError(hc.t, err)

		newVersion := &data.NodeVersion{
			BaseNodeVersion: data.BaseNodeVersion{
				OID:      id,
				ETag:     "12345678",
				FilePath: filepath,
			},
			IsUnversioned: true,
			IsCombined:    false,
		}

		_, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{
			"OID":           newVersion.OID.EncodeToString(),
			"FileName":      filename,
			"IsUnversioned": "true",
		}, false)
		require.NoError(hc.t, err)
		objects[i] = filepath
	}

	hc.treeMock.Sort()

	sort.Strings(objects)

	return objects
}

func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
	return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys)
}

func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *ListObjectsV2Response {
	query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
	if len(startAfter) != 0 {
		query.Add("start-after", startAfter)
	}
	if len(continuationToken) != 0 {
		query.Add("continuation-token", continuationToken)
	}
	if len(encodingType) != 0 {
		query.Add("encoding-type", encodingType)
	}

	w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
	hc.Handler().ListObjectsV2Handler(w, r)
	assertStatus(hc.t, w, http.StatusOK)
	res := &ListObjectsV2Response{}
	parseTestResponse(hc.t, w, res)
	return res
}

func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
	isTruncated, last bool, checkObjects, checkPrefixes []string) string {
	response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)

	require.Equal(t, isTruncated, response.IsTruncated)
	require.Equal(t, last, len(response.NextContinuationToken) == 0)

	require.Len(t, response.Contents, len(checkObjects))
	for i := 0; i < len(checkObjects); i++ {
		require.Equal(t, checkObjects[i], response.Contents[i].Key)
	}

	require.Len(t, response.CommonPrefixes, len(checkPrefixes))
	for i := 0; i < len(checkPrefixes); i++ {
		require.Equal(t, checkPrefixes[i], response.CommonPrefixes[i].Prefix)
	}

	return response.NextContinuationToken
}

func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Values {
	query := make(url.Values)

	if len(delimiter) != 0 {
		query.Add("delimiter", delimiter)
	}
	if len(prefix) != 0 {
		query.Add("prefix", prefix)
	}
	if maxKeys != -1 {
		query.Add("max-keys", strconv.Itoa(maxKeys))
	}

	return query
}

func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
	query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
	if len(marker) != 0 {
		query.Add("marker", marker)
	}

	w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
	hc.Handler().ListObjectsV1Handler(w, r)
	assertStatus(hc.t, w, http.StatusOK)
	res := &ListObjectsV1Response{}
	parseTestResponse(hc.t, w, res)
	return res
}

func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
	query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
	if len(keyMarker) != 0 {
		query.Add("key-marker", keyMarker)
	}
	if len(versionIDMarker) != 0 {
		query.Add("version-id-marker", versionIDMarker)
	}

	w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
	hc.Handler().ListBucketObjectVersionsHandler(w, r)
	assertStatus(hc.t, w, http.StatusOK)
	res := &ListObjectsVersionsResponse{}
	parseTestResponse(hc.t, w, res)
	return res
}