test_all: make a way of ignoring integration test failures

Use this to ignore known failures

fstest/test_all/config.go

@@ -28,11 +28,12 @@ type Test struct {
 //
 // FIXME make bucket based remotes set sub-dir automatically???
 type Backend struct {
-	Backend  string // name of the backend directory
+	Backend  string   // name of the backend directory
-	Remote   string // name of the test remote
+	Remote   string   // name of the test remote
-	SubDir   bool   // set to test with -sub-dir
+	SubDir   bool     // set to test with -sub-dir
-	FastList bool   // set to test with -fast-list
+	FastList bool     // set to test with -fast-list
-	OneOnly  bool   // set to run only one backend test at once
+	OneOnly  bool     // set to run only one backend test at once
+	Ignore   []string // test names to ignore the failure of
 }
 
 // MakeRuns creates Run objects the Backend and Test
@@ -48,6 +49,10 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 	if b.FastList && t.FastList {
 		fastlists = append(fastlists, true)
 	}
+	ignore := make(map[string]struct{}, len(b.Ignore))
+	for _, item := range b.Ignore {
+		ignore[item] = struct{}{}
+	}
 	for _, subdir := range subdirs {
 		for _, fastlist := range fastlists {
 			run := &Run{
@@ -59,6 +64,7 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 				NoRetries: t.NoRetries,
 				OneOnly:   b.OneOnly,
 				NoBinary:  t.NoBinary,
+				Ignore:    ignore,
 			}
 			if t.AddBackend {
 				run.Path = path.Join(run.Path, b.Backend)

fstest/test_all/config.yaml

@@ -65,10 +65,14 @@ backends:
    remote:   "TestS3DigitalOcean:"
    subdir:   true
    fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "s3"
    remote:   "TestS3Ceph:"
    subdir:   true
    fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "sftp"
    remote:   "TestSftp:"
    subdir:   false
@@ -81,6 +85,8 @@ backends:
    remote:   "TestSwiftCeph:"
    subdir:   true
    fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "yandex"
    remote:   "TestYandex:"
    subdir:   false
@@ -118,6 +124,8 @@ backends:
    remote:   "TestMega:"
    subdir:   false
    fastlist: false
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/PublicLink
  - backend:  "opendrive"
    remote:   "TestOpenDrive:"
    subdir:   false

fstest/test_all/run.go

@@ -45,6 +45,7 @@ type Run struct {
 	NoRetries bool   // don't retry if set
 	OneOnly   bool   // only run test for this backend at once
 	NoBinary  bool   // set to not build a binary
+	Ignore    map[string]struct{}
 	// Internals
 	cmdLine     []string
 	cmdString   string
@@ -138,9 +139,15 @@ func (r *Run) findFailures() {
 	oldFailedTests := r.failedTests
 	r.failedTests = nil
 	excludeParents := map[string]struct{}{}
+	ignored := 0
 	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
 		failedTest := string(matches[1])
-		r.failedTests = append(r.failedTests, failedTest)
+		// Skip any ignored failures
+		if _, found := r.Ignore[failedTest]; found {
+			ignored++
+		} else {
+			r.failedTests = append(r.failedTests, failedTest)
+		}
 		// Find all the parents of this test
 		parts := strings.Split(failedTest, "/")
 		for i := len(parts) - 1; i >= 1; i-- {
@@ -155,6 +162,12 @@ func (r *Run) findFailures() {
 		}
 	}
 	r.failedTests = newTests
+	if len(r.failedTests) == 0 && ignored > 0 {
+		log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored)
+		r.err = nil
+		r.dumpOutput()
+		return
+	}
 	if len(r.failedTests) != 0 {
 		r.runFlag = testsToRegexp(r.failedTests)
 	} else {