From dfadd9896999813397cf7457c01ffcc789162a56 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 26 Nov 2020 21:28:39 +0000 Subject: [PATCH] azureblob,memory,pcloud: fix setting of mime types Before this change the backend was reading the mime type of the destination object instead of the source object when uploading. This changes fixes the problem and introduces an integration test for it. See: https://forum.rclone.org/t/is-there-a-way-to-get-rclone-copy-to-preserve-metadata/20682/2 --- backend/azureblob/azureblob.go | 2 +- backend/memory/memory.go | 2 +- backend/pcloud/pcloud.go | 2 +- fstest/fstests/fstests.go | 340 +++++++++++++++++++-------------- 4 files changed, 195 insertions(+), 151 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 44345058d..8634d33f9 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1453,7 +1453,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op blob := o.getBlobReference() httpHeaders := azblob.BlobHTTPHeaders{} - httpHeaders.ContentType = fs.MimeType(ctx, o) + httpHeaders.ContentType = fs.MimeType(ctx, src) // Compute the Content-MD5 of the file, for multiparts uploads it // will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header // Note: If multipart, an MD5 checksum will also be computed for each uploaded block diff --git a/backend/memory/memory.go b/backend/memory/memory.go index cda1c0c28..0ff851d45 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -592,7 +592,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op data: data, hash: "", modTime: src.ModTime(ctx), - mimeType: fs.MimeType(ctx, o), + mimeType: fs.MimeType(ctx, src), } buckets.updateObjectData(bucket, bucketPath, o.od) return nil diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 0e564e038..58734e18e 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -1117,7 +1117,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Method: "PUT", Path: "/uploadfile", Body: in, - ContentType: fs.MimeType(ctx, o), + ContentType: fs.MimeType(ctx, src), ContentLength: &size, Parameters: url.Values{}, TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 9a13c43fa..622825afd 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -153,13 +153,32 @@ func retry(t *testing.T, what string, f func() error) { require.NoError(t, err, what) } -// testPut puts file with random contents to the remote -func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { - return PutTestContents(ctx, t, f, file, random.String(100), true) +// An fs.ObjectInfo that can override mime type +type objectInfoWithMimeType struct { + fs.ObjectInfo + mimeType string } -// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove -func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { +// Return a wrapped fs.ObjectInfo which returns the mime type given +func overrideMimeType(o fs.ObjectInfo, mimeType string) fs.ObjectInfo { + return &objectInfoWithMimeType{ + ObjectInfo: o, + mimeType: mimeType, + } +} + +// MimeType that was overridden +func (o *objectInfoWithMimeType) MimeType(ctx context.Context) string { + return o.mimeType +} + +// check interface +var _ fs.MimeTyper = (*objectInfoWithMimeType)(nil) + +// putTestContentsMimeType puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +// +// it uploads the object with the mimeType passed in if set +func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) (string, fs.Object) { var ( err error obj fs.Object @@ -172,6 +191,9 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It file.Size = int64(buf.Len()) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) + if mimeType != "" { + obji = overrideMimeType(obji, mimeType) + } obj, err = f.Put(ctx, in, obji) return err }) @@ -185,6 +207,21 @@ func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It return contents, obj } +// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { + return putTestContentsMimeType(ctx, t, f, file, contents, check, "") +} + +// testPut puts file with random contents to the remote +func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { + return PutTestContents(ctx, t, f, file, random.String(100), true) +} + +// testPutMimeType puts file with random contents to the remote and the mime type given +func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string) (string, fs.Object) { + return putTestContentsMimeType(ctx, t, f, file, random.String(100), true, mimeType) +} + // TestPutLarge puts file to the remote, checks it and removes it on success. func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) { var ( @@ -279,7 +316,7 @@ func stringsContains(x string, ss []string) bool { // - these are inside the "FsPutFiles" test. func Run(t *testing.T, opt *Opt) { var ( - remote fs.Fs + f fs.Fs remoteName = opt.RemoteName subRemoteName string subRemoteLeaf string @@ -288,6 +325,7 @@ func Run(t *testing.T, opt *Opt) { Path: "file name.txt", } file1Contents string + file1MimeType = "text/csv" file2 = fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, @@ -305,7 +343,7 @@ func Run(t *testing.T, opt *Opt) { // Skip the test if the remote isn't configured skipIfNotOk := func(t *testing.T) { - if remote == nil { + if f == nil { t.Skipf("WARN: %q not configured", remoteName) } } @@ -314,7 +352,7 @@ func Run(t *testing.T, opt *Opt) { // flag, returning a function to restore its value skipIfNotListR := func(t *testing.T) func() { skipIfNotOk(t) - if remote.Features().ListR == nil { + if f.Features().ListR == nil { t.Skip("FS has no ListR interface") } previous := ci.UseListR @@ -327,8 +365,8 @@ func Run(t *testing.T, opt *Opt) { // Skip if remote is not SetTier and GetTier capable skipIfNotSetTier := func(t *testing.T) { skipIfNotOk(t) - if remote.Features().SetTier == false || - remote.Features().GetTier == false { + if f.Features().SetTier == false || + f.Features().GetTier == false { t.Skip("FS has no SetTier & GetTier interfaces") } } @@ -374,7 +412,7 @@ func Run(t *testing.T, opt *Opt) { // remote - the result of fs.NewFs(TestRemote:subRemoteName) subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) require.NoError(t, err) - remote, err = fs.NewFs(context.Background(), subRemoteName) + f, err = fs.NewFs(context.Background(), subRemoteName) if err == fs.ErrorNotFoundInConfigFile { t.Logf("Didn't find %q in config file - skipping tests", remoteName) return @@ -390,7 +428,7 @@ func Run(t *testing.T, opt *Opt) { if opt.SkipFsCheckWrap { t.Skip("Skipping FsCheckWrap on this Fs") } - ft := new(fs.Features).Fill(ctx, remote) + ft := new(fs.Features).Fill(ctx, f) if ft.UnWrap == nil { t.Skip("Not a wrapping Fs") } @@ -418,7 +456,7 @@ func Run(t *testing.T, opt *Opt) { // Check to see if Fs advertises commands and they work and have docs t.Run("FsCommand", func(t *testing.T) { skipIfNotOk(t) - doCommand := remote.Features().Command + doCommand := f.Features().Command if doCommand == nil { t.Skip("No commands in this remote") } @@ -434,29 +472,29 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirNotFound tests deleting a non existent directory t.Run("FsRmdirNotFound", func(t *testing.T) { skipIfNotOk(t) - if isBucketBasedButNotRoot(remote) { + if isBucketBasedButNotRoot(f) { t.Skip("Skipping test as non root bucket based remote") } - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") assert.Error(t, err, "Expecting error on Rmdir non existent") }) // Make the directory - err = remote.Mkdir(ctx, "") + err = f.Mkdir(ctx, "") require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) // TestFsString tests the String method t.Run("FsString", func(t *testing.T) { skipIfNotOk(t) - str := remote.String() + str := f.String() require.NotEqual(t, "", str) }) // TestFsName tests the Name method t.Run("FsName", func(t *testing.T) { skipIfNotOk(t) - got := remote.Name() + got := f.Name() want := remoteName[:strings.LastIndex(remoteName, ":")+1] if isLocalRemote { want = "local:" @@ -467,8 +505,8 @@ func Run(t *testing.T, opt *Opt) { // TestFsRoot tests the Root method t.Run("FsRoot", func(t *testing.T) { skipIfNotOk(t) - name := remote.Name() + ":" - root := remote.Root() + name := f.Name() + ":" + root := f.Root() if isLocalRemote { // only check last path element on local require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root)) @@ -480,7 +518,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirEmpty tests deleting an empty directory t.Run("FsRmdirEmpty", func(t *testing.T) { skipIfNotOk(t) - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") require.NoError(t, err) }) @@ -490,41 +528,41 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsMkdir", func(t *testing.T) { skipIfNotOk(t) - err := remote.Mkdir(ctx, "") + err := f.Mkdir(ctx, "") require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) - err = remote.Mkdir(ctx, "") + err = f.Mkdir(ctx, "") require.NoError(t, err) // TestFsMkdirRmdirSubdir tests making and removing a sub directory t.Run("FsMkdirRmdirSubdir", func(t *testing.T) { skipIfNotOk(t) dir := "dir/subdir" - err := operations.Mkdir(ctx, remote, dir) + err := operations.Mkdir(ctx, f, dir) require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, f)) - err = operations.Rmdir(ctx, remote, dir) + err = operations.Rmdir(ctx, f, dir) require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, f)) - err = operations.Rmdir(ctx, remote, "dir") + err = operations.Rmdir(ctx, f, "dir") require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) }) // TestFsListEmpty tests listing an empty directory t.Run("FsListEmpty", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) }) // TestFsListDirEmpty tests listing the directories from an empty directory TestFsListDirEmpty := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, 1) - if !remote.Features().CanHaveEmptyDirectories { + objs, dirs, err := walk.GetAll(ctx, f, "", true, 1) + if !f.Features().CanHaveEmptyDirectories { if err != fs.ErrorDirNotFound { require.NoError(t, err) } @@ -545,8 +583,8 @@ func Run(t *testing.T, opt *Opt) { // TestFsListDirNotFound tests listing the directories from an empty directory TestFsListDirNotFound := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "does not exist", true, 1) - if !remote.Features().CanHaveEmptyDirectories { + objs, dirs, err := walk.GetAll(ctx, f, "does not exist", true, 1) + if !f.Features().CanHaveEmptyDirectories { if err != fs.ErrorDirNotFound { assert.NoError(t, err) assert.Equal(t, 0, len(objs)+len(dirs)) @@ -570,7 +608,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // check no files or dirs as pre-requisite - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) for _, test := range []struct { name string @@ -604,16 +642,16 @@ func Run(t *testing.T, opt *Opt) { fileName := encoder.Standard.Encode(test.path) dirName := fileName t.Logf("testing %q", fileName) - assert.NoError(t, remote.Mkdir(ctx, dirName)) + assert.NoError(t, f.Mkdir(ctx, dirName)) file := fstest.Item{ ModTime: time.Now(), Path: dirName + "/" + fileName, // test creating a file and dir with that name } - _, o := testPut(context.Background(), t, remote, &file) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, remote)) + _, o := testPut(context.Background(), t, f, &file) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, f)) assert.NoError(t, o.Remove(ctx)) - assert.NoError(t, remote.Rmdir(ctx, dirName)) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, remote)) + assert.NoError(t, f.Rmdir(ctx, dirName)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) }) } }) @@ -622,11 +660,11 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsNewObjectNotFound", func(t *testing.T) { skipIfNotOk(t) // Object in an existing directory - o, err := remote.NewObject(ctx, "potato") + o, err := f.NewObject(ctx, "potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) // Now try an object in a non existing directory - o, err = remote.NewObject(ctx, "directory/not/found/potato") + o, err = f.NewObject(ctx, "directory/not/found/potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) }) @@ -653,11 +691,11 @@ func Run(t *testing.T, opt *Opt) { in := io.MultiReader(buf, er) obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil) - _, err := remote.Put(ctx, in, obji) + _, err := f.Put(ctx, in, obji) // assert.Nil(t, obj) - FIXME some remotes return the object even on nil assert.NotNil(t, err) - obj, err := remote.NewObject(ctx, file2.Path) + obj, err := f.NewObject(ctx, file2.Path) assert.Nil(t, obj) assert.Equal(t, fs.ErrorObjectNotFound, err) }) @@ -665,7 +703,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsPutZeroLength", func(t *testing.T) { skipIfNotOk(t) - TestPutLarge(ctx, t, remote, &fstest.Item{ + TestPutLarge(ctx, t, f, &fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: fmt.Sprintf("zero-length-file"), Size: int64(0), @@ -674,7 +712,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("FsOpenWriterAt", func(t *testing.T) { skipIfNotOk(t) - openWriterAt := remote.Features().OpenWriterAt + openWriterAt := f.Features().OpenWriterAt if openWriterAt == nil { t.Skip("FS has no OpenWriterAt interface") } @@ -695,11 +733,11 @@ func Run(t *testing.T, opt *Opt) { assert.NoError(t, out.Close()) - obj := findObject(ctx, t, remote, path) + obj := findObject(ctx, t, f, path) assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ") assert.NoError(t, obj.Remove(ctx)) - assert.NoError(t, remote.Rmdir(ctx, "writer-at-subdir")) + assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) }) // TestFsChangeNotify tests that changes are properly @@ -710,12 +748,12 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have ChangeNotify - doChangeNotify := remote.Features().ChangeNotify + doChangeNotify := f.Features().ChangeNotify if doChangeNotify == nil { t.Skip("FS has no ChangeNotify interface") } - err := operations.Mkdir(ctx, remote, "dir") + err := operations.Mkdir(ctx, f, "dir") require.NoError(t, err) pollInterval := make(chan time.Duration) @@ -739,7 +777,7 @@ func Run(t *testing.T, opt *Opt) { var dirs []string for _, idx := range []int{1, 3, 2} { dir := fmt.Sprintf("dir/subdir%d", idx) - err = operations.Mkdir(ctx, remote, dir) + err = operations.Mkdir(ctx, f, dir) require.NoError(t, err) dirs = append(dirs, dir) } @@ -750,7 +788,7 @@ func Run(t *testing.T, opt *Opt) { ModTime: time.Now(), Path: fmt.Sprintf("dir/file%d", idx), } - _, o := testPut(ctx, t, remote, &file) + _, o := testPut(ctx, t, f, &file) objs = append(objs, o) } @@ -788,7 +826,7 @@ func Run(t *testing.T, opt *Opt) { } dirs = append(dirs, "dir") for _, dir := range dirs { - assert.NoError(t, remote.Rmdir(ctx, dir)) + assert.NoError(t, f.Rmdir(ctx, dir)) } }) @@ -797,9 +835,9 @@ func Run(t *testing.T, opt *Opt) { // Tests that require file1, file2 are within this t.Run("FsPutFiles", func(t *testing.T) { skipIfNotOk(t) - file1Contents, _ = testPut(ctx, t, remote, &file1) - /* file2Contents = */ testPut(ctx, t, remote, &file2) - file1Contents, _ = testPut(ctx, t, remote, &file1) + file1Contents, _ = testPut(ctx, t, f, &file1) + /* file2Contents = */ testPut(ctx, t, f, &file2) + file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType) // Note that the next test will check there are no duplicated file names // TestFsListDirFile2 tests the files are correctly uploaded by doing @@ -809,9 +847,9 @@ func Run(t *testing.T, opt *Opt) { list := func(dir string, expectedDirNames, expectedObjNames []string) { var objNames, dirNames []string for i := 1; i <= *fstest.ListRetries; i++ { - objs, dirs, err := walk.GetAll(ctx, remote, dir, true, 1) + objs, dirs, err := walk.GetAll(ctx, f, dir, true, 1) if errors.Cause(err) == fs.ErrorDirNotFound { - objs, dirs, err = walk.GetAll(ctx, remote, dir, true, 1) + objs, dirs, err = walk.GetAll(ctx, f, dir, true, 1) } require.NoError(t, err) objNames = objsToNames(objs) @@ -857,7 +895,7 @@ func Run(t *testing.T, opt *Opt) { // Test the files are all there with walk.ListR recursive listings t.Run("FsListR", func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, -1) + objs, dirs, err := walk.GetAll(ctx, f, "", true, -1) require.NoError(t, err) assert.Equal(t, []string{ "hello? sausage", @@ -875,7 +913,7 @@ func Run(t *testing.T, opt *Opt) { // walk.ListR recursive listings on a sub dir t.Run("FsListRSubdir", func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) + objs, dirs, err := walk.GetAll(ctx, f, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) require.NoError(t, err) assert.Equal(t, []string{ "hello? sausage/êé", @@ -914,7 +952,7 @@ func Run(t *testing.T, opt *Opt) { for i := 0; i < 2; i++ { dir, _ := path.Split(fileName) dir = dir[:len(dir)-1] - objs, dirs, err = walk.GetAll(ctx, remote, dir, true, -1) + objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1) } require.NoError(t, err) require.Len(t, objs, 1) @@ -932,7 +970,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsListLevel2 tests List works for 2 levels TestFsListLevel2 := func(t *testing.T) { skipIfNotOk(t) - objs, dirs, err := walk.GetAll(ctx, remote, "", true, 2) + objs, dirs, err := walk.GetAll(ctx, f, "", true, 2) if err == fs.ErrorLevelNotSupported { return } @@ -951,27 +989,27 @@ func Run(t *testing.T, opt *Opt) { // TestFsListFile1 tests file present t.Run("FsListFile1", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) }) // TestFsNewObject tests NewObject t.Run("FsNewObject", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) - file1.Check(t, obj, remote.Precision()) + obj := findObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) }) // TestFsListFile1and2 tests two files present t.Run("FsListFile1and2", func(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) }) // TestFsNewObjectDir tests NewObject on a directory which should produce an error t.Run("FsNewObjectDir", func(t *testing.T) { skipIfNotOk(t) dir := path.Dir(file2.Path) - obj, err := remote.NewObject(ctx, dir) + obj, err := f.NewObject(ctx, dir) assert.Nil(t, obj) assert.NotNil(t, err) }) @@ -981,7 +1019,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Purge - doPurge := remote.Features().Purge + doPurge := f.Features().Purge if doPurge == nil { t.Skip("FS has no Purge interface") } @@ -991,26 +1029,26 @@ func Run(t *testing.T, opt *Opt) { ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: "dirToPurge/fileToPurge.txt", } - _, _ = testPut(ctx, t, remote, &fileToPurge) + _, _ = testPut(ctx, t, f, &fileToPurge) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2, fileToPurge}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{ "dirToPurge", "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, fs.GetModifyWindow(ctx, remote)) + }, fs.GetModifyWindow(ctx, f)) // Now purge it - err = operations.Purge(ctx, remote, "dirToPurge") + err = operations.Purge(ctx, f, "dirToPurge") require.NoError(t, err) - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file1, file2}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, fs.GetModifyWindow(ctx, remote)) + }, fs.GetModifyWindow(ctx, f)) }) // TestFsCopy tests Copy @@ -1018,7 +1056,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Copy - doCopy := remote.Features().Copy + doCopy := f.Features().Copy if doCopy == nil { t.Skip("FS has no Copier interface") } @@ -1028,7 +1066,7 @@ func Run(t *testing.T, opt *Opt) { file2Copy.Path += "-copy" // do the copy - src := findObject(ctx, t, remote, file2.Path) + src := findObject(ctx, t, f, file2.Path) dst, err := doCopy(ctx, src, file2Copy.Path) if err == fs.ErrorCantCopy { t.Skip("FS can't copy") @@ -1036,7 +1074,7 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err, fmt.Sprintf("Error: %#v", err)) // check file exists in new listing - fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file2Copy}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2, file2Copy}) // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Copy.Path, dst.Remote()) @@ -1052,7 +1090,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have Move - doMove := remote.Features().Move + doMove := f.Features().Move if doMove == nil { t.Skip("FS has no Mover interface") } @@ -1067,14 +1105,14 @@ func Run(t *testing.T, opt *Opt) { // check happy path, i.e. no naming conflicts when rename and move are two // separate operations file2Move.Path = "other.txt" - src := findObject(ctx, t, remote, file2.Path) + src := findObject(ctx, t, f, file2.Path) dst, err := doMove(ctx, src, file2Move.Path) if err == fs.ErrorCantMove { t.Skip("FS can't move") } require.NoError(t, err) // check file exists in new listing - fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Move.Path, dst.Remote()) // 1: file name.txt @@ -1082,30 +1120,30 @@ func Run(t *testing.T, opt *Opt) { // Check conflict on "rename, then move" file1Move.Path = "moveTest/other.txt" - src = findObject(ctx, t, remote, file1.Path) + src = findObject(ctx, t, f, file1.Path) _, err = doMove(ctx, src, file1Move.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1Move, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1Move, file2Move}) // 1: moveTest/other.txt // 2: other.txt // Check conflict on "move, then rename" - src = findObject(ctx, t, remote, file1Move.Path) + src = findObject(ctx, t, f, file1Move.Path) _, err = doMove(ctx, src, file1.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2Move}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) // 1: file name.txt // 2: other.txt - src = findObject(ctx, t, remote, file2Move.Path) + src = findObject(ctx, t, f, file2Move.Path) _, err = doMove(ctx, src, file2.Path) require.NoError(t, err) - fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) // 1: file name.txt // 2: hello sausage?/../z.txt // Tidy up moveTest directory - require.NoError(t, remote.Rmdir(ctx, "moveTest")) + require.NoError(t, f.Rmdir(ctx, "moveTest")) }) // Move src to this remote using server-side move operations. @@ -1123,13 +1161,13 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have DirMove - doDirMove := remote.Features().DirMove + doDirMove := f.Features().DirMove if doDirMove == nil { t.Skip("FS has no DirMover interface") } // Check it can't move onto itself - err := doDirMove(ctx, remote, "", "") + err := doDirMove(ctx, f, "", "") require.Equal(t, fs.ErrorDirExists, err) // new remote @@ -1139,12 +1177,12 @@ func Run(t *testing.T, opt *Opt) { const newName = "new_name/sub_new_name" // try the move - err = newRemote.Features().DirMove(ctx, remote, "", newName) + err = newRemote.Features().DirMove(ctx, f, "", newName) require.NoError(t, err) // check remotes // remote should not exist here - _, err = remote.List(ctx, "") + _, err = f.List(ctx, "") assert.Equal(t, fs.ErrorDirNotFound, errors.Cause(err)) //fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision()) file1Copy := file1 @@ -1165,12 +1203,12 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err) // check remotes - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2, file1}, []string{ + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2, file1}, []string{ "hello? sausage", "hello? sausage/êé", "hello? sausage/êé/Hello, 世界", "hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", - }, remote.Precision()) + }, f.Precision()) fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{ "new_name", }, newRemote.Precision()) @@ -1179,17 +1217,17 @@ func Run(t *testing.T, opt *Opt) { // TestFsRmdirFull tests removing a non empty directory t.Run("FsRmdirFull", func(t *testing.T) { skipIfNotOk(t) - if isBucketBasedButNotRoot(remote) { + if isBucketBasedButNotRoot(f) { t.Skip("Skipping test as non root bucket based remote") } - err := remote.Rmdir(ctx, "") + err := f.Rmdir(ctx, "") require.Error(t, err, "Expecting error on RMdir on non empty remote") }) // TestFsPrecision tests the Precision of the Fs t.Run("FsPrecision", func(t *testing.T) { skipIfNotOk(t) - precision := remote.Precision() + precision := f.Precision() if precision == fs.ModTimeNotSupported { return } @@ -1202,7 +1240,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectString tests the Object String method t.Run("ObjectString", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Path, obj.String()) if opt.NilObject != nil { assert.Equal(t, "", opt.NilObject.String()) @@ -1212,13 +1250,13 @@ func Run(t *testing.T, opt *Opt) { // TestObjectFs tests the object can be found t.Run("ObjectFs", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) // If this is set we don't do the direct comparison of // the Fs from the object as it may be different if opt.SkipFsMatch { return } - testRemote := remote + testRemote := f if obj.Fs() != testRemote { // Check to see if this wraps something else if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil { @@ -1231,38 +1269,44 @@ func Run(t *testing.T, opt *Opt) { // TestObjectRemote tests the Remote is correct t.Run("ObjectRemote", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Path, obj.Remote()) }) // TestObjectHashes checks all the hashes the object supports t.Run("ObjectHashes", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) file1.CheckHashes(t, obj) }) // TestObjectModTime tests the ModTime of the object is correct TestObjectModTime := func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) - file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) + obj := findObject(ctx, t, f, file1.Path) + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) } t.Run("ObjectModTime", TestObjectModTime) // TestObjectMimeType tests the MimeType of the object is correct t.Run("ObjectMimeType", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + features := f.Features() + obj := findObject(ctx, t, f, file1.Path) do, ok := obj.(fs.MimeTyper) + require.Equal(t, features.ReadMimeType, ok, "mismatch between Object.MimeType and Features.ReadMimeType") if !ok { t.Skip("MimeType method not supported") } mimeType := do.MimeType(ctx) - if strings.ContainsRune(mimeType, ';') { - assert.Equal(t, "text/plain; charset=utf-8", mimeType) + if features.WriteMimeType { + assert.Equal(t, file1MimeType, mimeType, "can read and write mime types but failed") } else { - assert.Equal(t, "text/plain", mimeType) + if strings.ContainsRune(mimeType, ';') { + assert.Equal(t, "text/plain; charset=utf-8", mimeType) + } else { + assert.Equal(t, "text/plain", mimeType) + } } }) @@ -1270,7 +1314,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectSetModTime", func(t *testing.T) { skipIfNotOk(t) newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) err := obj.SetModTime(ctx, newModTime) if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete { t.Log(err) @@ -1278,7 +1322,7 @@ func Run(t *testing.T, opt *Opt) { } require.NoError(t, err) file1.ModTime = newModTime - file1.CheckModTime(t, obj, obj.ModTime(ctx), remote.Precision()) + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) // And make a new object and read it from there too TestObjectModTime(t) }) @@ -1286,21 +1330,21 @@ func Run(t *testing.T, opt *Opt) { // TestObjectSize tests that Size works t.Run("ObjectSize", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1.Size, obj.Size()) }) // TestObjectOpen tests that Open works t.Run("ObjectOpen", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ") }) // TestObjectOpenSeek tests that Open works with SeekOption t.Run("ObjectOpenSeek", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") }) @@ -1309,7 +1353,7 @@ func Run(t *testing.T, opt *Opt) { // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$' t.Run("ObjectOpenRange", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) for _, test := range []struct { ro fs.RangeOption wantStart, wantEnd int @@ -1330,7 +1374,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectPartialRead tests that reading only part of the object does the correct thing t.Run("ObjectPartialRead", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read") }) @@ -1343,18 +1387,18 @@ func Run(t *testing.T, opt *Opt) { in := io.TeeReader(buf, hash) file1.Size = int64(buf.Len()) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) obji := object.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs()) err := obj.Update(ctx, in, obji) require.NoError(t, err) file1.Hashes = hash.Sums() // check the object has been updated - file1.Check(t, obj, remote.Precision()) + file1.Check(t, obj, f.Precision()) // Re-read the object and check again - obj = findObject(ctx, t, remote, file1.Path) - file1.Check(t, obj, remote.Precision()) + obj = findObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) // check contents correct assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ") @@ -1364,7 +1408,7 @@ func Run(t *testing.T, opt *Opt) { // TestObjectStorable tests that Storable works t.Run("ObjectStorable", func(t *testing.T) { skipIfNotOk(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) require.NotNil(t, !obj.Storable(), "Expecting object to be storable") }) @@ -1398,7 +1442,7 @@ func Run(t *testing.T, opt *Opt) { // Test that things work from the root t.Run("FromRoot", func(t *testing.T) { - if features := remote.Features(); features.BucketBased && !features.BucketBasedRootOK { + if features := f.Features(); features.BucketBased && !features.BucketBasedRootOK { t.Skip("Can't list from root on this remote") } @@ -1500,7 +1544,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("PublicLink", func(t *testing.T) { skipIfNotOk(t) - doPublicLink := remote.Features().PublicLink + doPublicLink := f.Features().PublicLink if doPublicLink == nil { t.Skip("FS has no PublicLinker interface") } @@ -1561,7 +1605,7 @@ func Run(t *testing.T, opt *Opt) { // TestSetTier tests SetTier and GetTier functionality t.Run("SetTier", func(t *testing.T) { skipIfNotSetTier(t) - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) setter, ok := obj.(fs.SetTierer) assert.NotNil(t, ok) getter, ok := obj.(fs.GetTierer) @@ -1585,11 +1629,11 @@ func Run(t *testing.T, opt *Opt) { if opt.SkipObjectCheckWrap { t.Skip("Skipping FsCheckWrap on this Fs") } - ft := new(fs.Features).Fill(ctx, remote) + ft := new(fs.Features).Fill(ctx, f) if ft.UnWrap == nil { t.Skip("Not a wrapping Fs") } - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) _, unsupported := fs.ObjectOptionalInterfaces(obj) for _, name := range unsupported { if !stringsContains(name, opt.UnimplementableObjectMethods) { @@ -1602,11 +1646,11 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectRemove", func(t *testing.T) { skipIfNotOk(t) // remove file1 - obj := findObject(ctx, t, remote, file1.Path) + obj := findObject(ctx, t, f, file1.Path) err := obj.Remove(ctx) require.NoError(t, err) // check listing without modtime as TestPublicLink may change the modtime - fstest.CheckListingWithPrecision(t, remote, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) }) // TestAbout tests the About optional interface @@ -1614,7 +1658,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) // Check have About - doAbout := remote.Features().About + doAbout := f.Features().About if doAbout == nil { t.Skip("FS does not support About") } @@ -1633,7 +1677,7 @@ func Run(t *testing.T, opt *Opt) { // close to the end of suite. (See fs/operations/xtra_operations_test.go) t.Run("FsPutStream", func(t *testing.T) { skipIfNotOk(t) - if remote.Features().PutStream == nil { + if f.Features().PutStream == nil { t.Skip("FS has no PutStream interface") } @@ -1658,15 +1702,15 @@ func Run(t *testing.T, opt *Opt) { file.Size = -1 obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) - obj, err = remote.Features().PutStream(ctx, in, obji) + obj, err = f.Features().PutStream(ctx, in, obji) return err }) file.Hashes = uploadHash.Sums() file.Size = int64(contentSize) // use correct size when checking - file.Check(t, obj, remote.Precision()) + file.Check(t, obj, f.Precision()) // Re-read the object and check again - obj = findObject(ctx, t, remote, file.Path) - file.Check(t, obj, remote.Precision()) + obj = findObject(ctx, t, f, file.Path) + file.Check(t, obj, f.Precision()) require.NoError(t, obj.Remove(ctx)) }) } @@ -1675,10 +1719,10 @@ func Run(t *testing.T, opt *Opt) { // TestInternal calls InternalTest() on the Fs t.Run("Internal", func(t *testing.T) { skipIfNotOk(t) - if it, ok := remote.(InternalTester); ok { + if it, ok := f.(InternalTester); ok { it.InternalTest(t) } else { - t.Skipf("%T does not implement InternalTester", remote) + t.Skipf("%T does not implement InternalTester", f) } }) @@ -1693,12 +1737,12 @@ func Run(t *testing.T, opt *Opt) { t.Skip("not running with -short") } - setUploadChunkSizer, _ := remote.(SetUploadChunkSizer) + setUploadChunkSizer, _ := f.(SetUploadChunkSizer) if setUploadChunkSizer == nil { - t.Skipf("%T does not implement SetUploadChunkSizer", remote) + t.Skipf("%T does not implement SetUploadChunkSizer", f) } - setUploadCutoffer, _ := remote.(SetUploadCutoffer) + setUploadCutoffer, _ := f.(SetUploadCutoffer) minChunkSize := opt.ChunkedUpload.MinChunkSize if minChunkSize < 100 { @@ -1782,7 +1826,7 @@ func Run(t *testing.T, opt *Opt) { for _, fileSize := range testChunks { t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) { - TestPutLarge(ctx, t, remote, &fstest.Item{ + TestPutLarge(ctx, t, f, &fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()), Size: int64(fileSize), @@ -1810,7 +1854,7 @@ func Run(t *testing.T, opt *Opt) { in := bytes.NewBufferString(contents) obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil) - obj, err := remote.Put(ctx, in, obji) + obj, err := f.Put(ctx, in, obji) if err == nil { require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove") } @@ -1823,7 +1867,7 @@ func Run(t *testing.T, opt *Opt) { Path: "unknown-size-update.txt", } - testPut(ctx, t, remote, &unknownSizeUpdateFile) + testPut(ctx, t, f, &unknownSizeUpdateFile) defer func() { assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1") @@ -1832,7 +1876,7 @@ func Run(t *testing.T, opt *Opt) { newContents := random.String(200) in := bytes.NewBufferString(newContents) - obj := findObject(ctx, t, remote, unknownSizeUpdateFile.Path) + obj := findObject(ctx, t, f, unknownSizeUpdateFile.Path) obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs()) err := obj.Update(ctx, in, obji) if err == nil { @@ -1865,16 +1909,16 @@ func Run(t *testing.T, opt *Opt) { }) // Purge the folder - err = operations.Purge(ctx, remote, "") + err = operations.Purge(ctx, f, "") if errors.Cause(err) != fs.ErrorDirNotFound { require.NoError(t, err) } purged = true - fstest.CheckListing(t, remote, []fstest.Item{}) + fstest.CheckListing(t, f, []fstest.Item{}) // Check purging again if not bucket based - if !isBucketBasedButNotRoot(remote) { - err = operations.Purge(ctx, remote, "") + if !isBucketBasedButNotRoot(f) { + err = operations.Purge(ctx, f, "") assert.Error(t, err, "Expecting error after on second purge") if errors.Cause(err) != fs.ErrorDirNotFound { t.Log("Warning: this should produce fs.ErrorDirNotFound") @@ -1885,7 +1929,7 @@ func Run(t *testing.T, opt *Opt) { // Check directory is purged if !purged { - _ = operations.Purge(ctx, remote, "") + _ = operations.Purge(ctx, f, "") } // Remove the local directory so we don't clutter up /tmp