From 9f577563518113aa6201400efe46b1c2d4fe6559 Mon Sep 17 00:00:00 2001
From: Denis Kirillov <denis@nspcc.ru>
Date: Tue, 13 Jul 2021 19:58:55 +0300
Subject: [PATCH] [#155] Added s3 url encoder
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
---
api/handler/list.go | 14 ++---
api/handler/s3encoder.go | 111 ++++++++++++++++++++++++++++++++++
api/handler/s3encoder_test.go | 53 ++++++++++++++++
3 files changed, 171 insertions(+), 7 deletions(-)
create mode 100644 api/handler/s3encoder.go
create mode 100644 api/handler/s3encoder_test.go
diff --git a/api/handler/list.go b/api/handler/list.go
index bcf0ecde..81b681a0 100644
--- a/api/handler/list.go
+++ b/api/handler/list.go
@@ -193,14 +193,14 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
- Prefix: list.Prefixes[i],
+ Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
- Key: obj.Name,
+ Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
@@ -240,11 +240,11 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
res := &ListObjectsV2Response{
Name: arg.Bucket,
EncodingType: arg.Encode,
- Prefix: arg.Prefix,
+ Prefix: s3PathEncode(arg.Prefix, arg.Encode),
KeyCount: len(list.Objects) + len(list.Prefixes),
MaxKeys: arg.MaxKeys,
- Delimiter: arg.Delimiter,
- StartAfter: arg.StartAfter,
+ Delimiter: s3PathEncode(arg.Delimiter, arg.Encode),
+ StartAfter: s3PathEncode(arg.StartAfter, arg.Encode),
IsTruncated: list.IsTruncated,
@@ -255,14 +255,14 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
- Prefix: list.Prefixes[i],
+ Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
- Key: obj.Name,
+ Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
diff --git a/api/handler/s3encoder.go b/api/handler/s3encoder.go
new file mode 100644
index 00000000..3e205a81
--- /dev/null
+++ b/api/handler/s3encoder.go
@@ -0,0 +1,111 @@
+package handler
+
+import (
+ "strings"
+)
+
+type encoding int
+
+const (
+ encodePathSegment encoding = iota
+ encodeQueryComponent
+)
+
+const (
+ urlEncodingType = "url"
+ upperhex = "0123456789ABCDEF"
+)
+
+func shouldEscape(c byte) bool {
+ if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+ return false
+ }
+
+ switch c {
+ case '-', '_', '.', '/', '*':
+ return false
+ }
+ return true
+}
+
+// s3URLEncode is based on url.QueryEscape() code,
+// while considering some S3 exceptions.
+func s3URLEncode(s string, mode encoding) string {
+ spaceCount, hexCount := 0, 0
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ if shouldEscape(c) {
+ if c == ' ' && mode == encodeQueryComponent {
+ spaceCount++
+ } else {
+ hexCount++
+ }
+ }
+ }
+
+ if spaceCount == 0 && hexCount == 0 {
+ return s
+ }
+
+ var buf [64]byte
+ var t []byte
+
+ required := len(s) + 2*hexCount
+ if required <= len(buf) {
+ t = buf[:required]
+ } else {
+ t = make([]byte, required)
+ }
+
+ if hexCount == 0 {
+ copy(t, s)
+ for i := 0; i < len(s); i++ {
+ if s[i] == ' ' {
+ t[i] = '+'
+ }
+ }
+ return string(t)
+ }
+
+ j := 0
+ for i := 0; i < len(s); i++ {
+ switch c := s[i]; {
+ case c == ' ' && mode == encodeQueryComponent:
+ t[j] = '+'
+ j++
+ case shouldEscape(c):
+ t[j] = '%'
+ t[j+1] = upperhex[c>>4]
+ t[j+2] = upperhex[c&15]
+ j += 3
+ default:
+ t[j] = s[i]
+ j++
+ }
+ }
+ return string(t)
+}
+
+func s3QueryEncode(name string, encodingType string) (result string) {
+ if encodingType == "" {
+ return name
+ }
+ encodingType = strings.ToLower(encodingType)
+ switch encodingType {
+ case urlEncodingType:
+ return s3URLEncode(name, encodeQueryComponent)
+ }
+ return name
+}
+
+func s3PathEncode(name string, encodingType string) (result string) {
+ if encodingType == "" {
+ return name
+ }
+ encodingType = strings.ToLower(encodingType)
+ switch encodingType {
+ case urlEncodingType:
+ return s3URLEncode(name, encodePathSegment)
+ }
+ return name
+}
diff --git a/api/handler/s3encoder_test.go b/api/handler/s3encoder_test.go
new file mode 100644
index 00000000..da58cd88
--- /dev/null
+++ b/api/handler/s3encoder_test.go
@@ -0,0 +1,53 @@
+package handler
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestPathEncoder(t *testing.T) {
+ for _, tc := range []struct {
+ key string
+ expected string
+ }{
+ {key: "simple", expected: "simple"},
+ {key: "foo/bar", expected: "foo/bar"},
+ {key: "foo+1/bar", expected: "foo%2B1/bar"},
+ {key: "foo ab/bar", expected: "foo%20ab/bar"},
+ {key: "p-%", expected: "p-%25"},
+ {key: "p/", expected: "p/"},
+ {key: "p/", expected: "p/"},
+ {key: "~user", expected: "%7Euser"},
+ {key: "*user", expected: "*user"},
+ {key: "user+password", expected: "user%2Bpassword"},
+ {key: "_user", expected: "_user"},
+ {key: "firstname.lastname", expected: "firstname.lastname"},
+ } {
+ actual := s3PathEncode(tc.key, urlEncodingType)
+ require.Equal(t, tc.expected, actual)
+ }
+}
+
+func TestQueryEncoder(t *testing.T) {
+ for _, tc := range []struct {
+ key string
+ expected string
+ }{
+ {key: "simple", expected: "simple"},
+ {key: "foo/bar", expected: "foo/bar"},
+ {key: "foo+1/bar", expected: "foo%2B1/bar"},
+ {key: "foo ab/bar", expected: "foo+ab/bar"},
+ {key: "p-%", expected: "p-%25"},
+ {key: "p/", expected: "p/"},
+ {key: "p/", expected: "p/"},
+ {key: "~user", expected: "%7Euser"},
+ {key: "*user", expected: "*user"},
+ {key: "user+password", expected: "user%2Bpassword"},
+ {key: "_user", expected: "_user"},
+ {key: "firstname.lastname", expected: "firstname.lastname"},
+ } {
+ actual := s3QueryEncode(tc.key, urlEncodingType)
+ require.Equal(t, tc.expected, actual)
+ }
+}