rclone/fs/filter.go

// Control the filtering of files

package fs

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/spf13/pflag"
)

// Global
var (
	// Flags
	deleteExcluded = pflag.BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync")
	filterRule     = pflag.StringP("filter", "f", "", "Add a file-filtering rule")
	filterFrom     = pflag.StringP("filter-from", "", "", "Read filtering patterns from a file")
	excludeRule    = pflag.StringP("exclude", "", "", "Exclude files matching pattern")
	excludeFrom    = pflag.StringP("exclude-from", "", "", "Read exclude patterns from file")
	includeRule    = pflag.StringP("include", "", "", "Include files matching pattern")
	includeFrom    = pflag.StringP("include-from", "", "", "Read include patterns from file")
	filesFrom      = pflag.StringP("files-from", "", "", "Read list of source-file names from file")
	minSize        SizeSuffix
	maxSize        SizeSuffix
	dumpFilters    = pflag.BoolP("dump-filters", "", false, "Dump the filters to the output")
	//cvsExclude     = pflag.BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
)

func init() {
	pflag.VarP(&minSize, "min-size", "", "Don't transfer any file smaller than this in k or suffix k|M|G")
	pflag.VarP(&maxSize, "max-size", "", "Don't transfer any file larger than this in k or suffix k|M|G")
}

// rule is one filter rule
type rule struct {
	Include bool
	Regexp  *regexp.Regexp
}

// Match returns true if rule matches path
func (r *rule) Match(path string) bool {
	return r.Regexp.MatchString(path)
}

// String the rule
func (r *rule) String() string {
	c := "-"
	if r.Include {
		c = "+"
	}
	return fmt.Sprintf("%s %s", c, r.Regexp.String())
}

// filesMap describes the map of files to transfer
type filesMap map[string]struct{}

// Filter describes any filtering in operation
type Filter struct {
	DeleteExcluded bool
	MinSize        int64
	MaxSize        int64
	rules          []rule
	files          filesMap
}

// NewFilter parses the command line options and creates a Filter object
func NewFilter() (f *Filter, err error) {
	f = &Filter{
		DeleteExcluded: *deleteExcluded,
		MinSize:        int64(minSize),
		MaxSize:        int64(maxSize),
	}
	if *includeRule != "" {
		err = f.Add(true, *includeRule)
		if err != nil {
			return nil, err
		}
		// Add implicit exclude
		err = f.Add(false, "*")
		if err != nil {
			return nil, err
		}
	}
	if *includeFrom != "" {
		err := forEachLine(*includeFrom, func(line string) error {
			return f.Add(true, line)
		})
		if err != nil {
			return nil, err
		}
		// Add implicit exclude
		err = f.Add(false, "*")
		if err != nil {
			return nil, err
		}
	}
	if *excludeRule != "" {
		err = f.Add(false, *excludeRule)
		if err != nil {
			return nil, err
		}
	}
	if *excludeFrom != "" {
		err := forEachLine(*excludeFrom, func(line string) error {
			return f.Add(false, line)
		})
		if err != nil {
			return nil, err
		}
	}
	if *filterRule != "" {
		err = f.AddRule(*filterRule)
		if err != nil {
			return nil, err
		}
	}
	if *filterFrom != "" {
		err := forEachLine(*filterFrom, f.AddRule)
		if err != nil {
			return nil, err
		}
	}
	if *filesFrom != "" {
		err := forEachLine(*filesFrom, func(line string) error {
			return f.AddFile(line)
		})
		if err != nil {
			return nil, err
		}
	}
	if *dumpFilters {
		fmt.Println("--- start filters ---")
		fmt.Println(f.DumpFilters())
		fmt.Println("--- end filters ---")
	}
	return f, nil
}

// Add adds a filter rule with include or exclude status indicated
func (f *Filter) Add(Include bool, glob string) error {
	re, err := globToRegexp(glob)
	if err != nil {
		return err
	}
	rule := rule{
		Include: Include,
		Regexp:  re,
	}
	f.rules = append(f.rules, rule)
	return nil
}

// AddRule adds a filter rule with include/exclude indicated by the prefix
//
// These are
//
//   + glob
//   - glob
//   !
//
// '+' includes the glob, '-' excludes it and '!' resets the filter list
//
// Line comments may be introduced with '#' or ';'
func (f *Filter) AddRule(rule string) error {
	switch {
	case rule == "!":
		f.Clear()
		return nil
	case strings.HasPrefix(rule, "- "):
		return f.Add(false, rule[2:])
	case strings.HasPrefix(rule, "+ "):
		return f.Add(true, rule[2:])
	}
	return fmt.Errorf("Malformed rule %q", rule)
}

// AddFile adds a single file to the files from list
func (f *Filter) AddFile(file string) error {
	if f.files == nil {
		f.files = make(filesMap)
	}
	file = strings.Trim(file, "/")
	f.files[file] = struct{}{}
	return nil
}

// Clear clears all the filter rules
func (f *Filter) Clear() {
	f.rules = nil
}

// Include returns whether this object should be included into the
// sync or not
func (f *Filter) Include(remote string, size int64) bool {
	// filesFrom takes precedence
	if f.files != nil {
		_, include := f.files[remote]
		return include
	}
	if f.MinSize != 0 && size < f.MinSize {
		return false
	}
	if f.MaxSize != 0 && size > f.MaxSize {
		return false
	}
	for _, rule := range f.rules {
		if rule.Match(remote) {
			return rule.Include
		}
	}
	return true
}

// forEachLine calls fn on every line in the file pointed to by path
//
// It ignores empty lines and lines starting with '#' or ';'
func forEachLine(path string, fn func(string) error) (err error) {
	in, err := os.Open(path)
	if err != nil {
		return err
	}
	defer checkClose(in, &err)
	scanner := bufio.NewScanner(in)
	for scanner.Scan() {
		line := scanner.Text()
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' || line[0] == ';' {
			continue
		}
		err := fn(line)
		if err != nil {
			return err
		}
	}
	return scanner.Err()
}

// DumpFilters dumps the filters in textual form, 1 per line
func (f *Filter) DumpFilters() string {
	rules := []string{}
	for _, rule := range f.rules {
		rules = append(rules, rule.String())
	}
	return strings.Join(rules, "\n")
}