package handler import ( "fmt" "net/http" "net/url" "sort" "strconv" "strings" "testing" "time" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" "github.com/stretchr/testify/require" ) func TestParseContinuationToken(t *testing.T) { var err error t.Run("empty token", func(t *testing.T) { var queryValues = map[string][]string{ "continuation-token": {""}, } _, err = parseContinuationToken(queryValues) require.Error(t, err) }) t.Run("invalid not empty token", func(t *testing.T) { var queryValues = map[string][]string{ "continuation-token": {"asd"}, } _, err = parseContinuationToken(queryValues) require.Error(t, err) }) t.Run("valid token", func(t *testing.T) { tokenStr := "75BTT5Z9o79XuKdUeGqvQbqDnxu6qWcR5EhxW8BXFf8t" var queryValues = map[string][]string{ "continuation-token": {tokenStr}, } token, err := parseContinuationToken(queryValues) require.NoError(t, err) require.Equal(t, tokenStr, token) }) } func TestListObjectNullVersions(t *testing.T) { hc := prepareHandlerContext(t) bktName, objName := "bucket-versioning-enabled", "object" createTestBucket(hc, bktName) putObjectContent(hc, bktName, objName, "content") putBucketVersioning(t, hc, bktName, true) putObjectContent(hc, bktName, objName, "content2") result := listVersions(t, hc, bktName) require.Len(t, result.Version, 2) require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID) } func TestListObjectsPaging(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-versioning-enabled" createTestBucket(hc, bktName) n := 10 var objects []string for i := 0; i < n; i++ { objects = append(objects, "objects"+strconv.Itoa(i)) putObjectContent(hc, bktName, objects[i], "content") } sort.Strings(objects) result := &ListObjectsVersionsResponse{IsTruncated: true} for result.IsTruncated { result = listObjectsVersions(hc, bktName, "", "", result.NextKeyMarker, result.NextVersionIDMarker, n/3) for i, version := range result.Version { if objects[i] != version.Key { t.Errorf("expected: '%s', got: '%s'", objects[i], version.Key) } } objects = objects[len(result.Version):] } require.Empty(t, objects) } func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) { tc := prepareHandlerContext(t) bktName := "bucket-for-listing" objects := []string{"bar", "baz", "foo", "quxx"} bktInfo, _ := createBucketAndObject(tc, bktName, objects[0]) for _, objName := range objects[1:] { createTestObject(tc, bktInfo, objName, encryption.Params{}) } listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1) nextContinuationToken := listV2Response1.NextContinuationToken require.Equal(t, "baz", listV2Response1.Contents[0].Key) listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1) require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken) require.Equal(t, "bar", listV2Response2.StartAfter) require.False(t, listV2Response2.IsTruncated) require.Equal(t, "foo", listV2Response2.Contents[0].Key) require.Equal(t, "quxx", listV2Response2.Contents[1].Key) } func TestS3BucketListV2EncodingBasic(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing-v1-encoding" bktInfo := createTestBucket(hc, bktName) objects := []string{"foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b"} for _, objName := range objects { createTestObject(hc, bktInfo, objName, encryption.Params{}) } query := make(url.Values) query.Add("delimiter", "/") query.Add("encoding-type", "url") w, r := prepareTestFullRequest(hc, bktName, "", query, nil) hc.Handler().ListObjectsV2Handler(w, r) assertStatus(hc.t, w, http.StatusOK) listV2Response := &ListObjectsV2Response{} parseTestResponse(hc.t, w, listV2Response) require.Equal(t, "/", listV2Response.Delimiter) require.Len(t, listV2Response.Contents, 1) require.Equal(t, "asdf%2Bb", listV2Response.Contents[0].Key) require.Len(t, listV2Response.CommonPrefixes, 3) require.Equal(t, "foo%2B1/", listV2Response.CommonPrefixes[0].Prefix) require.Equal(t, "foo/", listV2Response.CommonPrefixes[1].Prefix) require.Equal(t, "quux%20ab/", listV2Response.CommonPrefixes[2].Prefix) } func TestS3BucketListDelimiterBasic(t *testing.T) { tc := prepareHandlerContext(t) bktName := "bucket-for-listing" objects := []string{"foo/bar", "foo/bar/xyzzy", "quux/thud", "asdf"} bktInfo, _ := createBucketAndObject(tc, bktName, objects[0]) for _, objName := range objects[1:] { createTestObject(tc, bktInfo, objName, encryption.Params{}) } listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1) require.Equal(t, "/", listV1Response.Delimiter) require.Equal(t, "asdf", listV1Response.Contents[0].Key) require.Len(t, listV1Response.CommonPrefixes, 2) require.Equal(t, "foo/", listV1Response.CommonPrefixes[0].Prefix) require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix) } func TestS3BucketListV2PrefixAlt(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"bar", "baz", "foo"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "ba", "", "", "", -1) require.Equal(t, "ba", response.Prefix) require.Len(t, response.Contents, 2) require.Equal(t, "bar", response.Contents[0].Key) require.Equal(t, "baz", response.Contents[1].Key) require.Empty(t, response.CommonPrefixes) } func TestS3BucketListV2PrefixNotExist(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"foo/bar", "foo/baz", "quux"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "d", "", "", "", -1) require.Equal(t, "d", response.Prefix) require.Empty(t, response.Contents) require.Empty(t, response.CommonPrefixes) } func TestS3BucketListV2PrefixUnreadable(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"foo/bar", "foo/baz", "quux"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "\x0a", "", "", "", -1) require.Equal(t, "\x0a", response.Prefix) require.Empty(t, response.Contents) require.Empty(t, response.CommonPrefixes) } func TestS3BucketListV2PrefixDelimiterAlt(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"bar", "bazar", "cab", "foo"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "ba", "a", "", "", -1) require.Equal(t, "ba", response.Prefix) require.Equal(t, "a", response.Delimiter) require.Len(t, response.Contents, 1) require.Equal(t, "bar", response.Contents[0].Key) require.Len(t, response.CommonPrefixes, 1) require.Equal(t, "baza", response.CommonPrefixes[0].Prefix) } func TestS3BucketListV2PrefixDelimiterDelimiterNotExist(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "b", "z", "", "", -1) require.Len(t, response.Contents, 3) require.Equal(t, "b/a/c", response.Contents[0].Key) require.Equal(t, "b/a/g", response.Contents[1].Key) require.Equal(t, "b/a/r", response.Contents[2].Key) require.Empty(t, response.CommonPrefixes) } func TestS3BucketListV2PrefixDelimiterPrefixDelimiterNotExist(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listing" createTestBucket(hc, bktName) objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"} for _, objName := range objects { putObject(hc, bktName, objName) } response := listObjectsV2(hc, bktName, "y", "z", "", "", -1) require.Empty(t, response.Contents) require.Empty(t, response.CommonPrefixes) } func TestS3BucketListV2DelimiterPercentage(t *testing.T) { tc := prepareHandlerContext(t) bktName := "bucket-for-listing" objects := []string{"b%ar", "b%az", "c%ab", "foo"} bktInfo, _ := createBucketAndObject(tc, bktName, objects[0]) for _, objName := range objects[1:] { createTestObject(tc, bktInfo, objName, encryption.Params{}) } listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1) require.Equal(t, "%", listV2Response.Delimiter) require.Len(t, listV2Response.Contents, 1) require.Equal(t, "foo", listV2Response.Contents[0].Key) require.Len(t, listV2Response.CommonPrefixes, 2) require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix) require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix) } func TestS3BucketListV2DelimiterPrefix(t *testing.T) { tc := prepareHandlerContext(t) bktName := "bucket-for-listingv2" objects := []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"} bktInfo, _ := createBucketAndObject(tc, bktName, objects[0]) for _, objName := range objects[1:] { createTestObject(tc, bktInfo, objName, encryption.Params{}) } var empty []string delim := "/" prefix := "" continuationToken := validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"asdf"}, empty) continuationToken = validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, true, false, empty, []string{"boo/"}) validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"cquux/"}) continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 2, true, false, []string{"asdf"}, []string{"boo/"}) validateListV2(t, tc, bktName, prefix, delim, continuationToken, 2, false, true, empty, []string{"cquux/"}) prefix = "boo/" continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"boo/bar"}, empty) validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"boo/baz/"}) validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"}) } func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) { hc := prepareHandlerContext(t) bktName, objName := "mint-bucket-for-listing-versions", "objName" createTestBucket(hc, bktName) putBucketVersioning(t, hc, bktName, true) length := 10 objects := make([]string, length) for i := 0; i < length; i++ { objects[i] = objName putObject(hc, bktName, objName) } maxKeys := 5 page1 := listObjectsVersions(hc, bktName, "", "", "", "", maxKeys) require.Len(t, page1.Version, maxKeys) checkVersionsNames(t, page1, objects) require.Equal(t, page1.Version[maxKeys-1].VersionID, page1.NextVersionIDMarker) require.True(t, page1.IsTruncated) require.Empty(t, page1.KeyMarker) require.Empty(t, page1.VersionIDMarker) page2 := listObjectsVersions(hc, bktName, "", "", page1.NextKeyMarker, page1.NextVersionIDMarker, maxKeys) require.Len(t, page2.Version, maxKeys) checkVersionsNames(t, page1, objects) require.Empty(t, page2.NextVersionIDMarker) require.False(t, page2.IsTruncated) require.Equal(t, page1.NextKeyMarker, page2.KeyMarker) require.Equal(t, page1.NextVersionIDMarker, page2.VersionIDMarker) } func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) { for i, v := range versions.Version { require.Equal(t, names[i], v.Key) } } func TestHugeListV2(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listingv2" bktInfo := createTestBucket(hc, bktName) objects := prepareObjects(hc, bktInfo, "", 50005) fmt.Println("listing start") start := time.Now() resp := &ListObjectsV2Response{IsTruncated: true} for resp.IsTruncated { resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1) for i, content := range resp.Contents { if content.Key != objects[i] { t.Errorf("expected '%s', got '%s'", objects[i], content.Key) } } objects = objects[len(resp.Contents):] } require.Empty(t, objects) fmt.Println(time.Since(start)) } func TestListV2StreamNested1(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listingv2-nested" bktInfo := createTestBucket(hc, bktName) objects1 := prepareObjects(hc, bktInfo, "prefix", 10) objects2 := prepareObjects(hc, bktInfo, "prefix2", 10) objects := append(objects1, objects2...) fmt.Println("listing start") start := time.Now() resp := &ListObjectsV2Response{IsTruncated: true} for resp.IsTruncated { resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1) for i, content := range resp.Contents { if content.Key != objects[i] { t.Errorf("expected '%s', got '%s'", objects[i], content.Key) } } objects = objects[len(resp.Contents):] } require.Empty(t, objects) fmt.Println(time.Since(start)) } func TestHugeListV1(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-listingv1" bktInfo := createTestBucket(hc, bktName) objects := prepareObjects(hc, bktInfo, "", 50005) fmt.Println("listing start") start := time.Now() resp := &ListObjectsV1Response{IsTruncated: true} for resp.IsTruncated { resp = listObjectsV1(hc, bktName, "", "", resp.NextMarker, -1) for i, content := range resp.Contents { if content.Key != objects[i] { t.Errorf("expected '%s', got '%s'", objects[i], content.Key) } } objects = objects[len(resp.Contents):] } require.Empty(t, objects) fmt.Println(time.Since(start)) } func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string { treeID := "version" parentID := uint64(0) if prefix != "" { for _, filename := range strings.Split(prefix, "/") { nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{ "FileName": filename, }) require.NoError(hc.t, err) parentID = nodeID } prefix += "/" } objects := make([]string, size) for i := range objects { filename := "object" + strconv.Itoa(i) filepath := prefix + filename prm := layer.PrmObjectCreate{ Container: bktInfo.CID, Filepath: filepath, Payload: nil, } id, err := hc.tp.CreateObject(hc.Context(), prm) require.NoError(hc.t, err) newVersion := &data.NodeVersion{ BaseNodeVersion: data.BaseNodeVersion{ OID: id, ETag: "12345678", FilePath: filepath, }, IsUnversioned: true, IsCombined: false, } _, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{ "OID": newVersion.OID.EncodeToString(), "FileName": filename, "IsUnversioned": "true", }, false) require.NoError(hc.t, err) objects[i] = filepath } hc.treeMock.Sort() sort.Strings(objects) return objects } func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response { return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys) } func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *ListObjectsV2Response { query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys) if len(startAfter) != 0 { query.Add("start-after", startAfter) } if len(continuationToken) != 0 { query.Add("continuation-token", continuationToken) } if len(encodingType) != 0 { query.Add("encoding-type", encodingType) } w, r := prepareTestFullRequest(hc, bktName, "", query, nil) hc.Handler().ListObjectsV2Handler(w, r) assertStatus(hc.t, w, http.StatusOK) res := &ListObjectsV2Response{} parseTestResponse(hc.t, w, res) return res } func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int, isTruncated, last bool, checkObjects, checkPrefixes []string) string { response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys) require.Equal(t, isTruncated, response.IsTruncated) require.Equal(t, last, len(response.NextContinuationToken) == 0) require.Len(t, response.Contents, len(checkObjects)) for i := 0; i < len(checkObjects); i++ { require.Equal(t, checkObjects[i], response.Contents[i].Key) } require.Len(t, response.CommonPrefixes, len(checkPrefixes)) for i := 0; i < len(checkPrefixes); i++ { require.Equal(t, checkPrefixes[i], response.CommonPrefixes[i].Prefix) } return response.NextContinuationToken } func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Values { query := make(url.Values) if len(delimiter) != 0 { query.Add("delimiter", delimiter) } if len(prefix) != 0 { query.Add("prefix", prefix) } if maxKeys != -1 { query.Add("max-keys", strconv.Itoa(maxKeys)) } return query } func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response { query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys) if len(marker) != 0 { query.Add("marker", marker) } w, r := prepareTestFullRequest(hc, bktName, "", query, nil) hc.Handler().ListObjectsV1Handler(w, r) assertStatus(hc.t, w, http.StatusOK) res := &ListObjectsV1Response{} parseTestResponse(hc.t, w, res) return res } func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse { query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys) if len(keyMarker) != 0 { query.Add("key-marker", keyMarker) } if len(versionIDMarker) != 0 { query.Add("version-id-marker", versionIDMarker) } w, r := prepareTestFullRequest(hc, bktName, "", query, nil) hc.Handler().ListBucketObjectVersionsHandler(w, r) assertStatus(hc.t, w, http.StatusOK) res := &ListObjectsVersionsResponse{} parseTestResponse(hc.t, w, res) return res }