9de485f949
This implements --auth-proxy for serve s3. In addition it: * add listbuckets tests with and without authProxy * use auth proxy test framework * servetest: implement workaround for #7454 * update github.com/rclone/gofakes3 to fix race condition
304 lines
6.9 KiB
Go
304 lines
6.9 KiB
Go
// Serve s3 tests set up a server and run the integration tests
|
|
// for the s3 remote against it.
|
|
|
|
package s3
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"path"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
"github.com/rclone/rclone/fs/object"
|
|
|
|
_ "github.com/rclone/rclone/backend/local"
|
|
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
|
|
"github.com/rclone/rclone/cmd/serve/servetest"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/config/configmap"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fstest"
|
|
httplib "github.com/rclone/rclone/lib/http"
|
|
"github.com/rclone/rclone/lib/random"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
endpoint = "localhost:0"
|
|
)
|
|
|
|
// Configure and serve the server
|
|
func serveS3(f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
|
|
keyid = random.String(16)
|
|
keysec = random.String(16)
|
|
serveropt := &Options{
|
|
HTTP: httplib.DefaultCfg(),
|
|
pathBucketMode: true,
|
|
hashName: "",
|
|
hashType: hash.None,
|
|
authPair: []string{fmt.Sprintf("%s,%s", keyid, keysec)},
|
|
}
|
|
|
|
serveropt.HTTP.ListenAddr = []string{endpoint}
|
|
w, _ = newServer(context.Background(), f, serveropt)
|
|
router := w.server.Router()
|
|
|
|
w.Bind(router)
|
|
_ = w.Serve()
|
|
testURL = w.server.URLs()[0]
|
|
|
|
return
|
|
}
|
|
|
|
// TestS3 runs the s3 server then runs the unit tests for the
|
|
// s3 remote against it.
|
|
func TestS3(t *testing.T) {
|
|
start := func(f fs.Fs) (configmap.Simple, func()) {
|
|
testURL, keyid, keysec, _ := serveS3(f)
|
|
// Config for the backend we'll use to connect to the server
|
|
config := configmap.Simple{
|
|
"type": "s3",
|
|
"provider": "Rclone",
|
|
"endpoint": testURL,
|
|
"access_key_id": keyid,
|
|
"secret_access_key": keysec,
|
|
}
|
|
|
|
return config, func() {}
|
|
}
|
|
|
|
servetest.Run(t, "s3", start)
|
|
}
|
|
|
|
// tests using the minio client
|
|
func TestEncodingWithMinioClient(t *testing.T) {
|
|
cases := []struct {
|
|
description string
|
|
bucket string
|
|
path string
|
|
filename string
|
|
expected string
|
|
}{
|
|
{
|
|
description: "weird file in bucket root",
|
|
bucket: "mybucket",
|
|
path: "",
|
|
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
|
|
},
|
|
{
|
|
description: "weird file inside a weird folder",
|
|
bucket: "mybucket",
|
|
path: "ä#/नेपाल&/?/",
|
|
filename: " file with w€r^d ch@r \\#~+§4%&'. txt ",
|
|
},
|
|
}
|
|
|
|
for _, tt := range cases {
|
|
t.Run(tt.description, func(t *testing.T) {
|
|
fstest.Initialise()
|
|
f, _, clean, err := fstest.RandomRemote()
|
|
assert.NoError(t, err)
|
|
defer clean()
|
|
err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
|
|
assert.NoError(t, err)
|
|
|
|
buf := bytes.NewBufferString("contents")
|
|
uploadHash := hash.NewMultiHasher()
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
obji := object.NewStaticObjectInfo(
|
|
path.Join(tt.bucket, tt.path, tt.filename),
|
|
time.Now(),
|
|
int64(buf.Len()),
|
|
true,
|
|
nil,
|
|
nil,
|
|
)
|
|
_, err = f.Put(context.Background(), in, obji)
|
|
assert.NoError(t, err)
|
|
|
|
endpoint, keyid, keysec, _ := serveS3(f)
|
|
testURL, _ := url.Parse(endpoint)
|
|
minioClient, err := minio.New(testURL.Host, &minio.Options{
|
|
Creds: credentials.NewStaticV4(keyid, keysec, ""),
|
|
Secure: false,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
buckets, err := minioClient.ListBuckets(context.Background())
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, buckets[0].Name, tt.bucket)
|
|
objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
|
|
Recursive: true,
|
|
})
|
|
for object := range objects {
|
|
assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type FileStuct struct {
|
|
path string
|
|
filename string
|
|
}
|
|
|
|
type TestCase struct {
|
|
description string
|
|
bucket string
|
|
files []FileStuct
|
|
keyID string
|
|
keySec string
|
|
shouldFail bool
|
|
}
|
|
|
|
func testListBuckets(t *testing.T, cases []TestCase, useProxy bool) {
|
|
fstest.Initialise()
|
|
|
|
var f fs.Fs
|
|
if useProxy {
|
|
// the backend config will be made by the proxy
|
|
prog, err := filepath.Abs("../servetest/proxy_code.go")
|
|
require.NoError(t, err)
|
|
files, err := filepath.Abs("testdata")
|
|
require.NoError(t, err)
|
|
cmd := "go run " + prog + " " + files
|
|
|
|
// FIXME: this is untidy setting a global variable!
|
|
proxyflags.Opt.AuthProxy = cmd
|
|
defer func() {
|
|
proxyflags.Opt.AuthProxy = ""
|
|
}()
|
|
|
|
f = nil
|
|
} else {
|
|
// create a test Fs
|
|
var err error
|
|
f, err = fs.NewFs(context.Background(), "testdata")
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
for _, tt := range cases {
|
|
t.Run(tt.description, func(t *testing.T) {
|
|
endpoint, keyid, keysec, s := serveS3(f)
|
|
defer func() {
|
|
assert.NoError(t, s.server.Shutdown())
|
|
}()
|
|
|
|
if tt.keyID != "" {
|
|
keyid = tt.keyID
|
|
}
|
|
if tt.keySec != "" {
|
|
keysec = tt.keySec
|
|
}
|
|
|
|
testURL, _ := url.Parse(endpoint)
|
|
minioClient, err := minio.New(testURL.Host, &minio.Options{
|
|
Creds: credentials.NewStaticV4(keyid, keysec, ""),
|
|
Secure: false,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
buckets, err := minioClient.ListBuckets(context.Background())
|
|
if tt.shouldFail {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, buckets)
|
|
assert.Equal(t, buckets[0].Name, tt.bucket)
|
|
|
|
o := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
|
|
Recursive: true,
|
|
})
|
|
// save files after reading from channel
|
|
objects := []string{}
|
|
for object := range o {
|
|
objects = append(objects, object.Key)
|
|
}
|
|
|
|
for _, tt := range tt.files {
|
|
file := path.Join(tt.path, tt.filename)
|
|
found := false
|
|
for _, fname := range objects {
|
|
if file == fname {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
require.Equal(t, true, found, "Object not found: "+file)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestListBuckets(t *testing.T) {
|
|
var cases = []TestCase{
|
|
{
|
|
description: "list buckets",
|
|
bucket: "mybucket",
|
|
files: []FileStuct{
|
|
{
|
|
path: "",
|
|
filename: "lorem.txt",
|
|
},
|
|
{
|
|
path: "foo",
|
|
filename: "bar.txt",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "list buckets: wrong s3 key",
|
|
bucket: "mybucket",
|
|
keyID: "invalid",
|
|
shouldFail: true,
|
|
},
|
|
{
|
|
description: "list buckets: wrong s3 secret",
|
|
bucket: "mybucket",
|
|
keySec: "invalid",
|
|
shouldFail: true,
|
|
},
|
|
}
|
|
|
|
testListBuckets(t, cases, false)
|
|
}
|
|
|
|
func TestListBucketsAuthProxy(t *testing.T) {
|
|
var cases = []TestCase{
|
|
{
|
|
description: "list buckets",
|
|
bucket: "mybucket",
|
|
// request with random keyid
|
|
// instead of what was set in 'authPair'
|
|
keyID: random.String(16),
|
|
files: []FileStuct{
|
|
{
|
|
path: "",
|
|
filename: "lorem.txt",
|
|
},
|
|
{
|
|
path: "foo",
|
|
filename: "bar.txt",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
description: "list buckets: wrong s3 secret",
|
|
bucket: "mybucket",
|
|
keySec: "invalid",
|
|
shouldFail: true,
|
|
},
|
|
}
|
|
|
|
testListBuckets(t, cases, true)
|
|
}
|