Merge pull request #157 from KirillovDenis/feature/155-s3_ceph_listObject_compatibility

[#155] Improved s3 listObjects compatibility
This commit is contained in:
Alex Vanin 2021-07-15 18:48:26 +03:00 committed by GitHub
commit 3a7876e292
7 changed files with 184 additions and 21 deletions

View file

@ -1633,7 +1633,7 @@ func GetAPIError(code ErrorCode) Error {
// getErrorResponse gets in standard error and resource value and // getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values. // provides a encodable populated response values.
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse { func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
code := "BadRequest" code := "InternalError"
desc := err.Error() desc := err.Error()
info := GetReqInfo(ctx) info := GetReqInfo(ctx)

View file

@ -37,7 +37,7 @@ type ListMultipartUploadsResult struct {
Xmlns string `xml:"xmlns,attr"` Xmlns string `xml:"xmlns,attr"`
} }
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, err error, logText string) { func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, err error, logText string) {
rid := api.GetRequestID(r.Context()) rid := api.GetRequestID(r.Context())
@ -119,12 +119,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
zap.String("request_id", rid), zap.String("request_id", rid),
zap.Error(err)) zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{ api.WriteErrorResponse(r.Context(), w, err, r.URL)
Code: api.GetAPIError(api.ErrBadRequest).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
return nil, nil, err return nil, nil, err
} }
@ -193,14 +188,14 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
// fill common prefixes // fill common prefixes
for i := range list.Prefixes { for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: list.Prefixes[i], Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
}) })
} }
// fill contents // fill contents
for _, obj := range list.Objects { for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{ res.Contents = append(res.Contents, Object{
Key: obj.Name, Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size, Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339), LastModified: obj.Created.Format(time.RFC3339),
@ -240,11 +235,11 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
res := &ListObjectsV2Response{ res := &ListObjectsV2Response{
Name: arg.Bucket, Name: arg.Bucket,
EncodingType: arg.Encode, EncodingType: arg.Encode,
Prefix: arg.Prefix, Prefix: s3PathEncode(arg.Prefix, arg.Encode),
KeyCount: len(list.Objects), KeyCount: len(list.Objects) + len(list.Prefixes),
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimiter, Delimiter: s3PathEncode(arg.Delimiter, arg.Encode),
StartAfter: arg.StartAfter, StartAfter: s3PathEncode(arg.StartAfter, arg.Encode),
IsTruncated: list.IsTruncated, IsTruncated: list.IsTruncated,
@ -255,14 +250,14 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
// fill common prefixes // fill common prefixes
for i := range list.Prefixes { for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: list.Prefixes[i], Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
}) })
} }
// fill contents // fill contents
for _, obj := range list.Objects { for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{ res.Contents = append(res.Contents, Object{
Key: obj.Name, Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size, Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339), LastModified: obj.Created.Format(time.RFC3339),
@ -287,7 +282,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
if r.URL.Query().Get("max-keys") == "" { if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 { } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys) return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
} }
@ -384,7 +379,7 @@ func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsP
if r.URL.Query().Get("max-keys") == "" { if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 { } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys) return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
} }

View file

@ -33,7 +33,7 @@ type ListObjectsV2Response struct {
KeyCount int KeyCount int
MaxKeys int MaxKeys int
Delimiter string Delimiter string `xml:"Delimiter,omitempty"`
// A flag that indicates whether or not ListObjects returned all of the results // A flag that indicates whether or not ListObjects returned all of the results
// that satisfied the search criteria. // that satisfied the search criteria.
IsTruncated bool IsTruncated bool
@ -75,7 +75,7 @@ type ListObjectsResponse struct {
NextMarker string `xml:"NextMarker,omitempty"` NextMarker string `xml:"NextMarker,omitempty"`
MaxKeys int MaxKeys int
Delimiter string Delimiter string `xml:"Delimiter,omitempty"`
// A flag that indicates whether or not ListObjects returned all of the results // A flag that indicates whether or not ListObjects returned all of the results
// that satisfied the search criteria. // that satisfied the search criteria.
IsTruncated bool IsTruncated bool

111
api/handler/s3encoder.go Normal file
View file

@ -0,0 +1,111 @@
package handler
import (
"strings"
)
type encoding int
const (
encodePathSegment encoding = iota
encodeQueryComponent
)
const (
urlEncodingType = "url"
upperhex = "0123456789ABCDEF"
)
func shouldEscape(c byte) bool {
if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
return false
}
switch c {
case '-', '_', '.', '/', '*':
return false
}
return true
}
// s3URLEncode is based on url.QueryEscape() code,
// while considering some S3 exceptions.
func s3URLEncode(s string, mode encoding) string {
spaceCount, hexCount := 0, 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c) {
if c == ' ' && mode == encodeQueryComponent {
spaceCount++
} else {
hexCount++
}
}
}
if spaceCount == 0 && hexCount == 0 {
return s
}
var buf [64]byte
var t []byte
required := len(s) + 2*hexCount
if required <= len(buf) {
t = buf[:required]
} else {
t = make([]byte, required)
}
if hexCount == 0 {
copy(t, s)
for i := 0; i < len(s); i++ {
if s[i] == ' ' {
t[i] = '+'
}
}
return string(t)
}
j := 0
for i := 0; i < len(s); i++ {
switch c := s[i]; {
case c == ' ' && mode == encodeQueryComponent:
t[j] = '+'
j++
case shouldEscape(c):
t[j] = '%'
t[j+1] = upperhex[c>>4]
t[j+2] = upperhex[c&15]
j += 3
default:
t[j] = s[i]
j++
}
}
return string(t)
}
func s3QueryEncode(name string, encodingType string) (result string) {
if encodingType == "" {
return name
}
encodingType = strings.ToLower(encodingType)
switch encodingType {
case urlEncodingType:
return s3URLEncode(name, encodeQueryComponent)
}
return name
}
func s3PathEncode(name string, encodingType string) (result string) {
if encodingType == "" {
return name
}
encodingType = strings.ToLower(encodingType)
switch encodingType {
case urlEncodingType:
return s3URLEncode(name, encodePathSegment)
}
return name
}

View file

@ -0,0 +1,53 @@
package handler
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestPathEncoder(t *testing.T) {
for _, tc := range []struct {
key string
expected string
}{
{key: "simple", expected: "simple"},
{key: "foo/bar", expected: "foo/bar"},
{key: "foo+1/bar", expected: "foo%2B1/bar"},
{key: "foo ab/bar", expected: "foo%20ab/bar"},
{key: "p-%", expected: "p-%25"},
{key: "p/", expected: "p/"},
{key: "p/", expected: "p/"},
{key: "~user", expected: "%7Euser"},
{key: "*user", expected: "*user"},
{key: "user+password", expected: "user%2Bpassword"},
{key: "_user", expected: "_user"},
{key: "firstname.lastname", expected: "firstname.lastname"},
} {
actual := s3PathEncode(tc.key, urlEncodingType)
require.Equal(t, tc.expected, actual)
}
}
func TestQueryEncoder(t *testing.T) {
for _, tc := range []struct {
key string
expected string
}{
{key: "simple", expected: "simple"},
{key: "foo/bar", expected: "foo/bar"},
{key: "foo+1/bar", expected: "foo%2B1/bar"},
{key: "foo ab/bar", expected: "foo+ab/bar"},
{key: "p-%", expected: "p-%25"},
{key: "p/", expected: "p/"},
{key: "p/", expected: "p/"},
{key: "~user", expected: "%7Euser"},
{key: "*user", expected: "*user"},
{key: "user+password", expected: "user%2Bpassword"},
{key: "_user", expected: "_user"},
{key: "firstname.lastname", expected: "firstname.lastname"},
} {
actual := s3QueryEncode(tc.key, urlEncodingType)
require.Equal(t, tc.expected, actual)
}
}

View file

@ -227,6 +227,10 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
uniqNames = make(map[string]bool) uniqNames = make(map[string]bool)
) )
if p.MaxKeys == 0 {
return &result, nil
}
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
return nil, err return nil, err
} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {

View file

@ -119,7 +119,7 @@ var s3ErrorResponseMap = map[string]string{
// WriteErrorResponse writes error headers. // WriteErrorResponse writes error headers.
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) { func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
code := http.StatusBadRequest code := http.StatusInternalServerError
if e, ok := err.(Error); ok { if e, ok := err.(Error); ok {
code = e.HTTPStatusCode code = e.HTTPStatusCode