fstest: remove -subdir flag as it no longer tests anything useful #3421

CONTRIBUTING.md

@@ -118,7 +118,7 @@ but they can be run against any of the remotes.
 
     cd fs/sync
     go test -v -remote TestDrive:
-    go test -v -remote TestDrive: -subdir
+    go test -v -remote TestDrive: -fast-list
 
     cd fs/operations
     go test -v -remote TestDrive:
@@ -362,9 +362,7 @@ Or if you want to run the integration tests manually:
       * `go test -v -remote TestRemote:`
       * `cd fs/sync`
       * `go test -v -remote TestRemote:`
-  * If you are making a bucket based remote, then check with this also
+  * If your remote defines `ListR` check with this also
-      * `go test -v -remote TestRemote: -subdir`
-  * And if your remote defines `ListR` this also
       * `go test -v -remote TestRemote: -fast-list`
 
 See the [testing](#testing) section for more information on integration tests.

cmd/mountlib/mounttest/fs.go

@@ -115,7 +115,7 @@ func newRun() *Run {
 	fstest.Initialise()
 
 	var err error
-	r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote()
 	if err != nil {
 		log.Fatalf("Failed to open remote %q: %v", *fstest.RemoteName, err)
 	}

cmd/serve/restic/restic_test.go

@@ -35,7 +35,7 @@ func TestRestic(t *testing.T) {
 
 	fstest.Initialise()
 
-	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	fremote, _, clean, err := fstest.RandomRemote()
 	assert.NoError(t, err)
 	defer clean()
 

cmd/serve/servetest/servetest.go

@@ -31,7 +31,7 @@ type StartFn func(f fs.Fs) (configmap.Simple, func())
 func run(t *testing.T, name string, start StartFn, useProxy bool) {
 	fstest.Initialise()
 
-	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	fremote, _, clean, err := fstest.RandomRemote()
 	assert.NoError(t, err)
 	defer clean()
 

fs/sync/sync_test.go

@@ -250,7 +250,7 @@ func TestServerSideCopy(t *testing.T) {
 	file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)
 	fstest.CheckItems(t, r.Fremote, file1)
 
-	FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote()
 	require.NoError(t, err)
 	defer finaliseCopy()
 	t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)
@@ -1026,7 +1026,7 @@ func TestSyncWithTrackRenames(t *testing.T) {
 
 // Test a server side move if possible, or the backup path if not
 func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmptyDirs bool) {
-	FremoteMove, _, finaliseMove, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	FremoteMove, _, finaliseMove, err := fstest.RandomRemote()
 	require.NoError(t, err)
 	defer finaliseMove()
 
@@ -1066,7 +1066,7 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
 	fstest.CheckItems(t, FremoteMove, file2, file1, file3u)
 
 	// Create a new empty remote for stuff to be moved into
-	FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote()
 	require.NoError(t, err)
 	defer finaliseMove2()
 

fstest/fstest.go

@@ -36,7 +36,6 @@ import (
 // Globals
 var (
 	RemoteName      = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
-	SubDir          = flag.Bool("subdir", false, "Set to test with a sub directory")
 	Verbose         = flag.Bool("verbose", false, "Set to enable logging")
 	DumpHeaders     = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)")
 	DumpBodies      = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)")
@@ -439,28 +438,22 @@ func RandomRemoteName(remoteName string) (string, string, error) {
 }
 
 // RandomRemote makes a random bucket or subdirectory on the remote
+// from the -remote parameter
 //
 // Call the finalise function returned to Purge the fs at the end (and
 // the parent if necessary)
 //
 // Returns the remote, its url, a finaliser and an error
-func RandomRemote(remoteName string, subdir bool) (fs.Fs, string, func(), error) {
+func RandomRemote() (fs.Fs, string, func(), error) {
 	var err error
 	var parentRemote fs.Fs
+	remoteName := *RemoteName
 
 	remoteName, _, err = RandomRemoteName(remoteName)
 	if err != nil {
 		return nil, "", nil, err
 	}
 
-	if subdir {
-		parentRemote, err = fs.NewFs(remoteName)
-		if err != nil {
-			return nil, "", nil, err
-		}
-		remoteName += "/rclone-test-subdir-" + random.String(8)
-	}
-
 	remote, err := fs.NewFs(remoteName)
 	if err != nil {
 		return nil, "", nil, err

fstest/fstests/fstests.go

@@ -349,6 +349,7 @@ func Run(t *testing.T, opt *Opt) {
 	if *fstest.RemoteName != "" {
 		remoteName = *fstest.RemoteName
 	}
+	fstest.RemoteName = &remoteName
 	t.Logf("Using remote %q", remoteName)
 	var err error
 	if remoteName == "" {
@@ -1117,7 +1118,7 @@ func Run(t *testing.T, opt *Opt) {
 				require.Equal(t, fs.ErrorDirExists, err)
 
 				// new remote
-				newRemote, _, removeNewRemote, err := fstest.RandomRemote(remoteName, false)
+				newRemote, _, removeNewRemote, err := fstest.RandomRemote()
 				require.NoError(t, err)
 				defer removeNewRemote()
 
@@ -1422,7 +1423,7 @@ func Run(t *testing.T, opt *Opt) {
 					require.NotEqual(t, "", link3, "Link should not be empty")
 
 					// sharing the "root" directory in a subremote
-					subRemote, _, removeSubRemote, err := fstest.RandomRemote(remoteName, false)
+					subRemote, _, removeSubRemote, err := fstest.RandomRemote()
 					require.NoError(t, err)
 					defer removeSubRemote()
 					// ensure sub remote isn't empty

fstest/run.go

@@ -90,7 +90,7 @@ func newRun() *Run {
 	Initialise()
 
 	var err error
-	r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote(*RemoteName, *SubDir)
+	r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote()
 	if err != nil {
 		r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
 	}

fstest/test_all/config.go

@@ -17,7 +17,6 @@ import (
 // Test describes an integration test to run with `go test`
 type Test struct {
 	Path       string // path to the source directory
-	SubDir     bool   // if it is possible to add -sub-dir to tests
 	FastList   bool   // if it is possible to add -fast-list to tests
 	AddBackend bool   // set if Path needs the current backend appending
 	NoRetries  bool   // set if no retries should be performed
@@ -31,7 +30,6 @@ type Test struct {
 type Backend struct {
 	Backend  string   // name of the backend directory
 	Remote   string   // name of the test remote
-	SubDir   bool     // set to test with -sub-dir
 	FastList bool     // set to test with -fast-list
 	OneOnly  bool     // set to run only one backend test at once
 	Ignore   []string // test names to ignore the failure of
@@ -54,16 +52,12 @@ func (b *Backend) includeTest(t *Test) bool {
 
 // MakeRuns creates Run objects the Backend and Test
 //
-// There can be several created, one for each combination of SubDir
+// There can be several created, one for each combination of optionl
-// and FastList
+// flags (eg FastList)
 func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 	if !b.includeTest(t) {
 		return runs
 	}
-	subdirs := []bool{false}
-	if b.SubDir && t.SubDir {
-		subdirs = append(subdirs, true)
-	}
 	fastlists := []bool{false}
 	if b.FastList && t.FastList {
 		fastlists = append(fastlists, true)
@@ -72,27 +66,24 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 	for _, item := range b.Ignore {
 		ignore[item] = struct{}{}
 	}
-	for _, subdir := range subdirs {
+	for _, fastlist := range fastlists {
-		for _, fastlist := range fastlists {
+		if t.LocalOnly && b.Backend != "local" {
-			if t.LocalOnly && b.Backend != "local" {
+			continue
-				continue
-			}
-			run := &Run{
-				Remote:    b.Remote,
-				Backend:   b.Backend,
-				Path:      t.Path,
-				SubDir:    subdir,
-				FastList:  fastlist,
-				NoRetries: t.NoRetries,
-				OneOnly:   b.OneOnly,
-				NoBinary:  t.NoBinary,
-				Ignore:    ignore,
-			}
-			if t.AddBackend {
-				run.Path = path.Join(run.Path, b.Backend)
-			}
-			runs = append(runs, run)
 		}
+		run := &Run{
+			Remote:    b.Remote,
+			Backend:   b.Backend,
+			Path:      t.Path,
+			FastList:  fastlist,
+			NoRetries: t.NoRetries,
+			OneOnly:   b.OneOnly,
+			NoBinary:  t.NoBinary,
+			Ignore:    ignore,
+		}
+		if t.AddBackend {
+			run.Path = path.Join(run.Path, b.Backend)
+		}
+		runs = append(runs, run)
 	}
 	return runs
 }

fstest/test_all/config.yaml

@@ -3,10 +3,8 @@ tests:
    addbackend: true
    nobinary:   true
  - path: fs/operations
-   subdir:   true
    fastlist: true
  - path: fs/sync
-   subdir:   true
    fastlist: true
  - path: vfs
  - path: cmd/serve/restic
@@ -14,35 +12,27 @@ tests:
 backends:
  # - backend:  "amazonclouddrive"
  #   remote:   "TestAmazonCloudDrive:"
- #   subdir:   false
  #   fastlist: false
  - backend:  "local"
    remote:   ""
-   subdir:   false
    fastlist: false
  - backend:  "b2"
    remote:   "TestB2:"
-   subdir:   true
    fastlist: true
  - backend:  "crypt"
    remote:   "TestCryptDrive:"
-   subdir:   false
    fastlist: true
  - backend:  "crypt"
    remote:   "TestCryptSwift:"
-   subdir:   false
    fastlist: false
  - backend:  "drive"
    remote:   "TestDrive:"
-   subdir:   false
    fastlist: true
  - backend:  "dropbox"
    remote:   "TestDropbox:"
-   subdir:   false
    fastlist: false
  - backend:  "googlecloudstorage"
    remote:   "TestGoogleCloudStorage:"
-   subdir:   true
    fastlist: true
  - backend:  "googlephotos"
    remote:   "TestGooglePhotos:"
@@ -50,114 +40,88 @@ backends:
      - backend
  - backend:  "hubic"
    remote:   "TestHubic:"
-   subdir:   false
    fastlist: false
  - backend:  "jottacloud"
    remote:   "TestJottacloud:"
-   subdir:   false
    fastlist: true
  - backend:  "onedrive"
    remote:   "TestOneDrive:"
-   subdir:   false
    fastlist: false
  - backend:  "s3"
    remote:   "TestS3:"
-   subdir:   true
    fastlist: true
  - backend:  "s3"
    remote:   "TestS3Minio:"
-   subdir:   true
    fastlist: true
  - backend:  "s3"
    remote:   "TestS3Wasabi:"
-   subdir:   true
    fastlist: true
  - backend:  "s3"
    remote:   "TestS3DigitalOcean:"
-   subdir:   true
    fastlist: true
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "s3"
    remote:   "TestS3Ceph:"
-   subdir:   true
    fastlist: true
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
-   subdir:   true
    fastlist: true
  - backend:  "sftp"
    remote:   "TestSftp:"
-   subdir:   false
    fastlist: false
  - backend:  "swift"
    remote:   "TestSwift:"
-   subdir:   true
    fastlist: true
  - backend:  "swift"
    remote:   "TestSwiftCeph:"
-   subdir:   true
    fastlist: true
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "yandex"
    remote:   "TestYandex:"
-   subdir:   false
    fastlist: false
  - backend:  "ftp"
    remote:   "TestFTP:"
-   subdir:   false
    fastlist: false
  - backend:  "box"
    remote:   "TestBox:"
-   subdir:   false
    fastlist: false
  - backend:  "fichier"
    remote:   "TestFichier:"
-   subdir:   false
    fastlist: false
  - backend:  "qingstor"
    remote:   "TestQingStor:"
-   subdir:   false
    fastlist: false
    oneonly:  true
  - backend:  "azureblob"
    remote:   "TestAzureBlob:"
-   subdir:   true
    fastlist: true
  - backend:  "pcloud"
    remote:   "TestPcloud:"
-   subdir:   false
    fastlist: false
  - backend:  "webdav"
    remote:   "TestWebdav:"
-   subdir:   false
    fastlist: false
  - backend:  "cache"
    remote:   "TestCache:"
-   subdir:   false
    fastlist: false
  - backend:  "mega"
    remote:   "TestMega:"
-   subdir:   false
    fastlist: false
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/PublicLink
  - backend:  "opendrive"
    remote:   "TestOpenDrive:"
-   subdir:   false
    fastlist: false
  - backend:  "union"
    remote:   "TestUnion:"
-   subdir:   false
    fastlist: false
  - backend:  "koofr"
    remote:   "TestKoofr:"
-   subdir:   false
    fastlist: false
  - backend:  "premiumizeme"
    remote:   "TestPremiumizeMe:"
-   subdir:   false
    fastlist: false

fstest/test_all/report.go

@@ -208,7 +208,6 @@ a:focus {
 <th>Backend</th>
 <th>Remote</th>
 <th>Test</th>
-<th>SubDir</th>
 <th>FastList</th>
 <th>Failed</th>
 <th>Logs</th>
@@ -220,7 +219,6 @@ a:focus {
 <td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td>
 <td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
 <td>{{ .Path }}</td>
-<td><span class="{{ .SubDir }}">{{ .SubDir }}</span></td>
 <td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
 <td>{{ .FailedTests }}</td>
 <td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>

fstest/test_all/run.go

@@ -38,7 +38,6 @@ type Run struct {
 	Remote    string // name of the test remote
 	Backend   string // name of the backend
 	Path      string // path to the source directory
-	SubDir    bool   // add -sub-dir to tests
 	FastList  bool   // add -fast-list to tests
 	NoRetries bool   // don't retry if set
 	OneOnly   bool   // only run test for this backend at once
@@ -80,11 +79,6 @@ func (rs Runs) Less(i, j int) bool {
 	} else if a.Path > b.Path {
 		return false
 	}
-	if !a.SubDir && b.SubDir {
-		return true
-	} else if a.SubDir && !b.SubDir {
-		return false
-	}
 	if !a.FastList && b.FastList {
 		return true
 	} else if a.FastList && !b.FastList {
@@ -311,9 +305,6 @@ func (r *Run) Name() string {
 		strings.Replace(r.Path, "/", ".", -1),
 		r.Remote,
 	}
-	if r.SubDir {
-		ns = append(ns, "subdir")
-	}
 	if r.FastList {
 		ns = append(ns, "fastlist")
 	}
@@ -341,9 +332,6 @@ func (r *Run) Init() {
 	if *runOnly != "" {
 		r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly)
 	}
-	if r.SubDir {
-		r.cmdLine = append(r.cmdLine, "-subdir")
-	}
 	if r.FastList {
 		r.cmdLine = append(r.cmdLine, "-fast-list")
 	}