Add option to exclude directories with a tagfile

The option is named --exclude-if-present and accepts a parameter
filename[:content]. Directories are excluded and their contents is not
backed up if they contain a file with the specified name and,
optionally, that starts with the specified content. The tagfile itself
is never excluded.

There is also a shortcut --exclude-caches that works in the same way as
the likewise-named option of tar(1): Directories are recognized as cache
if they contain a file named "CACHEDIR.TAG.

Closes #317.

CHANGELOG.md

@@ -72,6 +72,14 @@ Small changes
    run. This is now corrected.
    https://github.com/restic/restic/pull/1191
 
+ * A new option `--exclude-caches` was added that allows excluding cache
+   directories (that are tagged as such). This is a special case of a more
+   generic option `--exclude-if-present` which excludes a directory if a file
+   with a specific name (and contents) is present.
+   https://github.com/restic/restic/issues/317
+   https://github.com/restic/restic/pull/1170
+
+
 Important Changes in 0.7.1
 ==========================
 

cmd/restic/cmd_backup.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -53,16 +54,18 @@ given as the arguments.
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	Parent         string
+	Parent           string
-	Force          bool
+	Force            bool
-	Excludes       []string
+	Excludes         []string
-	ExcludeFiles   []string
+	ExcludeFiles     []string
-	ExcludeOtherFS bool
+	ExcludeOtherFS   bool
-	Stdin          bool
+	ExcludeIfPresent string
-	StdinFilename  string
+	ExcludeCaches    bool
-	Tags           []string
+	Stdin            bool
-	Hostname       string
+	StdinFilename    string
-	FilesFrom      string
+	Tags             []string
+	Hostname         string
+	FilesFrom        string
 }
 
 var backupOptions BackupOptions
@@ -76,6 +79,8 @@ func init() {
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
+	f.StringVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", "", "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided")
+	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`)
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
 	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
@@ -416,6 +421,18 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...)
 	}
 
+	if opts.ExcludeCaches {
+		if opts.ExcludeIfPresent != "" {
+			return fmt.Errorf("cannot have --exclude-caches defined at the same time as --exclude-if-present")
+		}
+		opts.ExcludeIfPresent = "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55"
+	}
+
+	excludeByFile, err := excludeByFile(opts.ExcludeIfPresent)
+	if err != nil {
+		return err
+	}
+
 	selectFilter := func(item string, fi os.FileInfo) bool {
 		matched, _, err := filter.List(opts.Excludes, item)
 		if err != nil {
@@ -427,6 +444,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 			return false
 		}
 
+		if excludeByFile(item) {
+			debug.Log("path %q excluded by tagfile", item)
+			return false
+		}
+
 		if !opts.ExcludeOtherFS || fi == nil {
 			return true
 		}
@@ -522,3 +544,86 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string {
 	}
 	return excludes
 }
+
+// FilenameCheck is a function that takes a filename and returns a boolean
+// depending on arbitrary check.
+type FilenameCheck func(filename string) bool
+
+// excludeByFile returns a FilenameCheck which itself returns whether a path
+// should be excluded. The FilenameCheck considers a file to be excluded when
+// it resides in a directory with an exclusion file, that is specified by
+// excludeFileSpec in the form "filename[:content]". The returned error is
+// non-nil if the filename component of excludeFileSpec is empty.
+func excludeByFile(excludeFileSpec string) (FilenameCheck, error) {
+	if excludeFileSpec == "" {
+		return func(string) bool { return false }, nil
+	}
+	colon := strings.Index(excludeFileSpec, ":")
+	if colon == 0 {
+		return nil, fmt.Errorf("no name for exclusion tagfile provided")
+	}
+	tf, tc := "", ""
+	if colon > 0 {
+		tf = excludeFileSpec[:colon]
+		tc = excludeFileSpec[colon+1:]
+	} else {
+		tf = excludeFileSpec
+	}
+	debug.Log("using %q as exclusion tagfile", tf)
+	fn := func(filename string) bool {
+		return isExcludedByFile(filename, tf, tc)
+	}
+	return fn, nil
+}
+
+// isExcludedByFile interprets filename as a path and returns true if that file
+// is in a excluded directory. A directory is identified as excluded if it contains a
+// tagfile which bears the name specified in tagFilename and starts with header.
+func isExcludedByFile(filename, tagFilename, header string) bool {
+	if tagFilename == "" {
+		return false
+	}
+	dir, base := filepath.Split(filename)
+	if base == tagFilename {
+		return false // do not exclude the tagfile itself
+	}
+	tf := filepath.Join(dir, tagFilename)
+	_, err := fs.Lstat(tf)
+	if os.IsNotExist(err) {
+		return false
+	}
+	if err != nil {
+		Warnf("could not access exclusion tagfile: %v", err)
+		return false
+	}
+	// when no signature is given, the mere presence of tf is enough reason
+	// to exclude filename
+	if len(header) == 0 {
+		return true
+	}
+	// From this stage, errors mean tagFilename exists but it is malformed.
+	// Warnings will be generated so that the user is informed that the
+	// indented ignore-action is not performed.
+	f, err := os.Open(tf)
+	if err != nil {
+		Warnf("could not open exclusion tagfile: %v", err)
+		return false
+	}
+	defer f.Close()
+	buf := make([]byte, len(header))
+	_, err = io.ReadFull(f, buf)
+	// EOF is handled with a dedicated message, otherwise the warning were too cryptic
+	if err == io.EOF {
+		Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf)
+		return false
+	}
+	if err != nil {
+		Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err)
+		return false
+	}
+	if bytes.Compare(buf, []byte(header)) != 0 {
+		Warnf("invalid signature in exclusion tagfile %q\n", tf)
+		return false
+	}
+	return true
+}

cmd/restic/exclude_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestIsExcludedByFile(t *testing.T) {
+	const (
+		tagFilename = "CACHEDIR.TAG"
+		header      = "Signature: 8a477f597d28d172789f06886806bc55"
+	)
+	tests := []struct {
+		name    string
+		tagFile string
+		content string
+		want    bool
+	}{
+		{"NoTagfile", "", "", false},
+		{"EmptyTagfile", tagFilename, "", true},
+		{"UnnamedTagFile", "", header, false},
+		{"WrongTagFile", "notatagfile", header, false},
+		{"IncorrectSig", tagFilename, header[1:], false},
+		{"ValidSig", tagFilename, header, true},
+		{"ValidPlusStuff", tagFilename, header + "foo", true},
+		{"ValidPlusNewlineAndStuff", tagFilename, header + "\nbar", true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tempDir, err := ioutil.TempDir("", "restic-test-")
+			if err != nil {
+				t.Fatalf("could not create temp dir: %v", err)
+			}
+			defer os.RemoveAll(tempDir)
+			foo := filepath.Join(tempDir, "foo")
+			err = ioutil.WriteFile(foo, []byte("foo"), 0666)
+			if err != nil {
+				t.Fatalf("could not write file: %v", err)
+			}
+			if tc.tagFile != "" {
+				tagFile := filepath.Join(tempDir, tc.tagFile)
+				err = ioutil.WriteFile(tagFile, []byte(tc.content), 0666)
+				if err != nil {
+					t.Fatalf("could not write tagfile: %v", err)
+				}
+			}
+			h := header
+			if tc.content == "" {
+				h = ""
+			}
+			if got := isExcludedByFile(foo, tagFilename, h); tc.want != got {
+				t.Fatalf("expected %v, got %v", tc.want, got)
+			}
+		})
+	}
+}

doc/man/restic-backup.1

@@ -24,10 +24,18 @@ given as the arguments.
 \fB\-e\fP, \fB\-\-exclude\fP=[]
     exclude a \fB\fCpattern\fR (can be specified multiple times)
 
+.PP
+\fB\-\-exclude\-caches\fP[=false]
+    excludes cache directories that are marked with a CACHEDIR.TAG file
+
 .PP
 \fB\-\-exclude\-file\fP=[]
     read exclude patterns from a \fB\fCfile\fR (can be specified multiple times)
 
+.PP
+\fB\-\-exclude\-if\-present\fP=""
+    takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided
+
 .PP
 \fB\-\-files\-from\fP=""
     read the files to backup from file (can be combined with file args)