serve s3: fix file name encoding using s3 serve with mc client

using the mc (minio) client file encoding were wrong
see Mikubill/gofakes3#2 for details

cmd/serve/s3/s3_test.go

@@ -4,16 +4,24 @@
 package s3
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"math/rand"
+	"net/url"
 	"os"
 	"os/exec"
+	"path"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/rclone/rclone/fs/object"
+
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/cmd/serve/servetest"
 	"github.com/rclone/rclone/fs"
@@ -29,43 +37,27 @@ const (
 	endpoint = "localhost:0"
 )
 
-// TestS3 runs the s3 server then runs the unit tests for the
-// s3 remote against it.
-func TestS3(t *testing.T) {
-	// Configure and start the server
-	start := func(f fs.Fs) (configmap.Simple, func()) {
-		keyid := RandString(16)
-		keysec := RandString(16)
-		serveropt := &Options{
-			HTTP:           httplib.DefaultCfg(),
-			pathBucketMode: true,
-			hashName:       "",
-			hashType:       hash.None,
-			authPair:       []string{fmt.Sprintf("%s,%s", keyid, keysec)},
-		}
-
-		serveropt.HTTP.ListenAddr = []string{endpoint}
-		w, err := newServer(context.Background(), f, serveropt)
-		router := w.Router()
-		assert.NoError(t, err)
-
-		w.Bind(router)
-		w.Serve()
-		testURL := w.Server.URLs()[0]
-		// Config for the backend we'll use to connect to the server
-		config := configmap.Simple{
-			"type":              "s3",
-			"provider":          "Rclone",
-			"endpoint":          testURL,
-			"list_url_encode":   "true",
-			"access_key_id":     keyid,
-			"secret_access_key": keysec,
-		}
-
-		return config, func() {}
+// Configure and serve the server
+func serveS3(f fs.Fs) (testURL string, keyid string, keysec string) {
+	keyid = RandString(16)
+	keysec = RandString(16)
+	serveropt := &Options{
+		HTTP:           httplib.DefaultCfg(),
+		pathBucketMode: true,
+		hashName:       "",
+		hashType:       hash.None,
+		authPair:       []string{fmt.Sprintf("%s,%s", keyid, keysec)},
 	}
 
-	Run(t, "s3", start)
+	serveropt.HTTP.ListenAddr = []string{endpoint}
+	w, _ := newServer(context.Background(), f, serveropt)
+	router := w.Router()
+
+	w.Bind(router)
+	w.Serve()
+	testURL = w.Server.URLs()[0]
+
+	return
 }
 
 func RandString(n int) string {
@@ -79,7 +71,28 @@ func RandString(n int) string {
 	return hex.EncodeToString(b)[:n]
 }
 
-func Run(t *testing.T, name string, start servetest.StartFn) {
+// TestS3 runs the s3 server then runs the unit tests for the
+// s3 remote against it.
+func TestS3(t *testing.T) {
+	start := func(f fs.Fs) (configmap.Simple, func()) {
+		testURL, keyid, keysec := serveS3(f)
+		// Config for the backend we'll use to connect to the server
+		config := configmap.Simple{
+			"type":              "s3",
+			"provider":          "Rclone",
+			"endpoint":          testURL,
+			"list_url_encode":   "true",
+			"access_key_id":     keyid,
+			"secret_access_key": keysec,
+		}
+
+		return config, func() {}
+	}
+
+	RunS3UnitTests(t, "s3", start)
+}
+
+func RunS3UnitTests(t *testing.T, name string, start servetest.StartFn) {
 	fstest.Initialise()
 	ci := fs.GetConfig(context.Background())
 	ci.DisableFeatures = append(ci.DisableFeatures, "Metadata")
@@ -105,7 +118,7 @@ func Run(t *testing.T, name string, start servetest.StartFn) {
 		require.NoError(t, os.Chdir(cwd))
 	}()
 
-	// Run the backend tests with an on the fly remote
+	// RunS3UnitTests the backend tests with an on the fly remote
 	args := []string{"test"}
 	if testing.Verbose() {
 		args = append(args, "-v")
@@ -126,11 +139,79 @@ func Run(t *testing.T, name string, start servetest.StartFn) {
 		cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v)
 	}
 
-	// Run the test
+	// RunS3UnitTests the test
 	out, err := cmd.CombinedOutput()
 	if len(out) != 0 {
 		t.Logf("\n----------\n%s----------\n", string(out))
 	}
 	assert.NoError(t, err, "Running "+name+" integration tests")
+}
+
+// tests using the minio client
+func TestEncodingWithMinioClient(t *testing.T) {
+	cases := []struct {
+		description string
+		bucket      string
+		path        string
+		filename    string
+		expected    string
+	}{
+		{
+			description: "weird file in bucket root",
+			bucket:      "mybucket",
+			path:        "",
+			filename:    " file with w€r^d ch@r \\#~+§4%&'. txt ",
+		},
+		{
+			description: "weird file inside a weird folder",
+			bucket:      "mybucket",
+			path:        "ä#/नेपाल&/?/",
+			filename:    " file with w€r^d ch@r \\#~+§4%&'. txt ",
+		},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.description, func(t *testing.T) {
+			fstest.Initialise()
+			f, _, clean, err := fstest.RandomRemote()
+			assert.NoError(t, err)
+			defer clean()
+			err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
+			assert.NoError(t, err)
+
+			buf := bytes.NewBufferString("contents")
+			uploadHash := hash.NewMultiHasher()
+			in := io.TeeReader(buf, uploadHash)
+
+			obji := object.NewStaticObjectInfo(
+				path.Join(tt.bucket, tt.path, tt.filename),
+				time.Now(),
+				int64(buf.Len()),
+				true,
+				nil,
+				nil,
+			)
+			_, err = f.Put(context.Background(), in, obji)
+			assert.NoError(t, err)
+
+			endpoint, keyid, keysec := serveS3(f)
+			testURL, _ := url.Parse(endpoint)
+			minioClient, err := minio.New(testURL.Host, &minio.Options{
+				Creds:  credentials.NewStaticV4(keyid, keysec, ""),
+				Secure: false,
+			})
+			assert.NoError(t, err)
+
+			buckets, err := minioClient.ListBuckets(context.Background())
+			assert.NoError(t, err)
+			assert.Equal(t, buckets[0].Name, tt.bucket)
+			objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
+				Recursive: true,
+			})
+			for object := range objects {
+				assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
+			}
+		})
+	}
 
 }