s3: fix encoding for control characters - Fixes #3345

backend/s3/s3.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"regexp"
 	"strings"
@@ -1221,11 +1222,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	for {
 		// FIXME need to implement ALL loop
 		req := s3.ListObjectsInput{
-			Bucket:    &bucket,
+			Bucket:       &bucket,
-			Delimiter: &delimiter,
+			Delimiter:    &delimiter,
-			Prefix:    &directory,
+			Prefix:       &directory,
-			MaxKeys:   &maxKeys,
+			MaxKeys:      &maxKeys,
-			Marker:    marker,
+			Marker:       marker,
+			EncodingType: aws.String(s3.EncodingTypeUrl),
 		}
 		var resp *s3.ListObjectsOutput
 		var err error
@@ -1259,6 +1261,11 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					continue
 				}
 				remote := *commonPrefix.Prefix
+				remote, err = url.QueryUnescape(remote)
+				if err != nil {
+					fs.Logf(f, "failed to URL decode %q in listing common prefix: %v", *commonPrefix.Prefix, err)
+					continue
+				}
 				if !strings.HasPrefix(remote, prefix) {
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
@@ -1278,6 +1285,11 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 		for _, object := range resp.Contents {
 			remote := aws.StringValue(object.Key)
+			remote, err = url.QueryUnescape(remote)
+			if err != nil {
+				fs.Logf(f, "failed to URL decode %q in listing: %v", aws.StringValue(object.Key), err)
+				continue
+			}
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", remote)
 				continue