Merge pull request #157 from KirillovDenis/feature/155-s3_ceph_listObject_compatibility

[#155] Improved s3 listObjects compatibility

api/errors.go

@@ -1633,7 +1633,7 @@ func GetAPIError(code ErrorCode) Error {
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values.
 func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
-	code := "BadRequest"
+	code := "InternalError"
 	desc := err.Error()
 
 	info := GetReqInfo(ctx)

api/handler/list.go

@@ -37,7 +37,7 @@ type ListMultipartUploadsResult struct {
 	Xmlns   string   `xml:"xmlns,attr"`
 }
 
-var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 
 func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, err error, logText string) {
 	rid := api.GetRequestID(r.Context())
@@ -119,12 +119,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
 			zap.String("request_id", rid),
 			zap.Error(err))
 
-		api.WriteErrorResponse(r.Context(), w, api.Error{
+		api.WriteErrorResponse(r.Context(), w, err, r.URL)
-			Code:           api.GetAPIError(api.ErrBadRequest).Code,
-			Description:    err.Error(),
-			HTTPStatusCode: http.StatusBadRequest,
-		}, r.URL)
-
 		return nil, nil, err
 	}
 
@@ -193,14 +188,14 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
 	// fill common prefixes
 	for i := range list.Prefixes {
 		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
-			Prefix: list.Prefixes[i],
+			Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
 		})
 	}
 
 	// fill contents
 	for _, obj := range list.Objects {
 		res.Contents = append(res.Contents, Object{
-			Key:          obj.Name,
+			Key:          s3PathEncode(obj.Name, arg.Encode),
 			Size:         obj.Size,
 			LastModified: obj.Created.Format(time.RFC3339),
 
@@ -240,11 +235,11 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
 	res := &ListObjectsV2Response{
 		Name:         arg.Bucket,
 		EncodingType: arg.Encode,
-		Prefix:       arg.Prefix,
+		Prefix:       s3PathEncode(arg.Prefix, arg.Encode),
-		KeyCount:     len(list.Objects),
+		KeyCount:     len(list.Objects) + len(list.Prefixes),
 		MaxKeys:      arg.MaxKeys,
-		Delimiter:    arg.Delimiter,
+		Delimiter:    s3PathEncode(arg.Delimiter, arg.Encode),
-		StartAfter:   arg.StartAfter,
+		StartAfter:   s3PathEncode(arg.StartAfter, arg.Encode),
 
 		IsTruncated: list.IsTruncated,
 
@@ -255,14 +250,14 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
 	// fill common prefixes
 	for i := range list.Prefixes {
 		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
-			Prefix: list.Prefixes[i],
+			Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
 		})
 	}
 
 	// fill contents
 	for _, obj := range list.Objects {
 		res.Contents = append(res.Contents, Object{
-			Key:          obj.Name,
+			Key:          s3PathEncode(obj.Name, arg.Encode),
 			Size:         obj.Size,
 			LastModified: obj.Created.Format(time.RFC3339),
 
@@ -287,7 +282,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
 
 	if r.URL.Query().Get("max-keys") == "" {
 		res.MaxKeys = maxObjectList
-	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 {
+	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
 		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
 	}
 
@@ -384,7 +379,7 @@ func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsP
 
 	if r.URL.Query().Get("max-keys") == "" {
 		res.MaxKeys = maxObjectList
-	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 {
+	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
 		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
 	}
 

api/handler/response.go

@@ -33,7 +33,7 @@ type ListObjectsV2Response struct {
 
 	KeyCount  int
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -75,7 +75,7 @@ type ListObjectsResponse struct {
 	NextMarker string `xml:"NextMarker,omitempty"`
 
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool

api/handler/s3encoder.go

@@ -0,0 +1,111 @@
+package handler
+
+import (
+	"strings"
+)
+
+type encoding int
+
+const (
+	encodePathSegment encoding = iota
+	encodeQueryComponent
+)
+
+const (
+	urlEncodingType = "url"
+	upperhex        = "0123456789ABCDEF"
+)
+
+func shouldEscape(c byte) bool {
+	if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '_', '.', '/', '*':
+		return false
+	}
+	return true
+}
+
+// s3URLEncode is based on url.QueryEscape() code,
+// while considering some S3 exceptions.
+func s3URLEncode(s string, mode encoding) string {
+	spaceCount, hexCount := 0, 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			if c == ' ' && mode == encodeQueryComponent {
+				spaceCount++
+			} else {
+				hexCount++
+			}
+		}
+	}
+
+	if spaceCount == 0 && hexCount == 0 {
+		return s
+	}
+
+	var buf [64]byte
+	var t []byte
+
+	required := len(s) + 2*hexCount
+	if required <= len(buf) {
+		t = buf[:required]
+	} else {
+		t = make([]byte, required)
+	}
+
+	if hexCount == 0 {
+		copy(t, s)
+		for i := 0; i < len(s); i++ {
+			if s[i] == ' ' {
+				t[i] = '+'
+			}
+		}
+		return string(t)
+	}
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case c == ' ' && mode == encodeQueryComponent:
+			t[j] = '+'
+			j++
+		case shouldEscape(c):
+			t[j] = '%'
+			t[j+1] = upperhex[c>>4]
+			t[j+2] = upperhex[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+	return string(t)
+}
+
+func s3QueryEncode(name string, encodingType string) (result string) {
+	if encodingType == "" {
+		return name
+	}
+	encodingType = strings.ToLower(encodingType)
+	switch encodingType {
+	case urlEncodingType:
+		return s3URLEncode(name, encodeQueryComponent)
+	}
+	return name
+}
+
+func s3PathEncode(name string, encodingType string) (result string) {
+	if encodingType == "" {
+		return name
+	}
+	encodingType = strings.ToLower(encodingType)
+	switch encodingType {
+	case urlEncodingType:
+		return s3URLEncode(name, encodePathSegment)
+	}
+	return name
+}

api/handler/s3encoder_test.go

@@ -0,0 +1,53 @@
+package handler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPathEncoder(t *testing.T) {
+	for _, tc := range []struct {
+		key      string
+		expected string
+	}{
+		{key: "simple", expected: "simple"},
+		{key: "foo/bar", expected: "foo/bar"},
+		{key: "foo+1/bar", expected: "foo%2B1/bar"},
+		{key: "foo ab/bar", expected: "foo%20ab/bar"},
+		{key: "p-%", expected: "p-%25"},
+		{key: "p/", expected: "p/"},
+		{key: "p/", expected: "p/"},
+		{key: "~user", expected: "%7Euser"},
+		{key: "*user", expected: "*user"},
+		{key: "user+password", expected: "user%2Bpassword"},
+		{key: "_user", expected: "_user"},
+		{key: "firstname.lastname", expected: "firstname.lastname"},
+	} {
+		actual := s3PathEncode(tc.key, urlEncodingType)
+		require.Equal(t, tc.expected, actual)
+	}
+}
+
+func TestQueryEncoder(t *testing.T) {
+	for _, tc := range []struct {
+		key      string
+		expected string
+	}{
+		{key: "simple", expected: "simple"},
+		{key: "foo/bar", expected: "foo/bar"},
+		{key: "foo+1/bar", expected: "foo%2B1/bar"},
+		{key: "foo ab/bar", expected: "foo+ab/bar"},
+		{key: "p-%", expected: "p-%25"},
+		{key: "p/", expected: "p/"},
+		{key: "p/", expected: "p/"},
+		{key: "~user", expected: "%7Euser"},
+		{key: "*user", expected: "*user"},
+		{key: "user+password", expected: "user%2Bpassword"},
+		{key: "_user", expected: "_user"},
+		{key: "firstname.lastname", expected: "firstname.lastname"},
+	} {
+		actual := s3QueryEncode(tc.key, urlEncodingType)
+		require.Equal(t, tc.expected, actual)
+	}
+}

api/layer/layer.go

@@ -227,6 +227,10 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		uniqNames = make(map[string]bool)
 	)
 
+	if p.MaxKeys == 0 {
+		return &result, nil
+	}
+
 	if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
 		return nil, err
 	} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {

api/response.go

@@ -119,7 +119,7 @@ var s3ErrorResponseMap = map[string]string{
 
 // WriteErrorResponse writes error headers.
 func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
-	code := http.StatusBadRequest
+	code := http.StatusInternalServerError
 
 	if e, ok := err.(Error); ok {
 		code = e.HTTPStatusCode