rclone/fstest/test_all/config.go
Nick Craig-Wood e43b5ce5e5 Remove github.com/pkg/errors and replace with std library version
This is possible now that we no longer support go1.12 and brings
rclone into line with standard practices in the Go world.

This also removes errors.New and errors.Errorf from lib/errors and
prefers the stdlib errors package over lib/errors.
2021-11-07 11:53:30 +00:00

191 lines
5 KiB
Go

// Config handling
package main
import (
"fmt"
"io/ioutil"
"log"
"path"
"github.com/rclone/rclone/fs"
yaml "gopkg.in/yaml.v2"
)
// Test describes an integration test to run with `go test`
type Test struct {
Path string // path to the source directory
FastList bool // if it is possible to add -fast-list to tests
Short bool // if it is possible to run the test with -short
AddBackend bool // set if Path needs the current backend appending
NoRetries bool // set if no retries should be performed
NoBinary bool // set to not build a binary in advance
LocalOnly bool // if set only run with the local backend
}
// Backend describes a backend test
//
// FIXME make bucket-based remotes set sub-dir automatically???
type Backend struct {
Backend string // name of the backend directory
Remote string // name of the test remote
FastList bool // set to test with -fast-list
Short bool // set to test with -short
OneOnly bool // set to run only one backend test at once
MaxFile string // file size limit
CleanUp bool // when running clean, run cleanup first
Ignore []string // test names to ignore the failure of
Tests []string // paths of tests to run, blank for all
ListRetries int // -list-retries if > 0
ExtraTime float64 // factor to multiply the timeout by
}
// includeTest returns true if this backend should be included in this
// test
func (b *Backend) includeTest(t *Test) bool {
if len(b.Tests) == 0 {
return true
}
for _, testPath := range b.Tests {
if testPath == t.Path {
return true
}
}
return false
}
// MakeRuns creates Run objects the Backend and Test
//
// There can be several created, one for each combination of optional
// flags (e.g. FastList)
func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
if !b.includeTest(t) {
return runs
}
maxSize := fs.SizeSuffix(0)
if b.MaxFile != "" {
if err := maxSize.Set(b.MaxFile); err != nil {
log.Printf("Invalid maxfile value %q: %v", b.MaxFile, err)
}
}
fastlists := []bool{false}
if b.FastList && t.FastList {
fastlists = append(fastlists, true)
}
ignore := make(map[string]struct{}, len(b.Ignore))
for _, item := range b.Ignore {
ignore[item] = struct{}{}
}
for _, fastlist := range fastlists {
if t.LocalOnly && b.Backend != "local" {
continue
}
run := &Run{
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
FastList: fastlist,
Short: (b.Short && t.Short),
NoRetries: t.NoRetries,
OneOnly: b.OneOnly,
NoBinary: t.NoBinary,
SizeLimit: int64(maxSize),
Ignore: ignore,
ListRetries: b.ListRetries,
ExtraTime: b.ExtraTime,
}
if t.AddBackend {
run.Path = path.Join(run.Path, b.Backend)
}
runs = append(runs, run)
}
return runs
}
// Config describes the config for this program
type Config struct {
Tests []Test
Backends []Backend
}
// NewConfig reads the config file
func NewConfig(configFile string) (*Config, error) {
d, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
config := &Config{}
err = yaml.Unmarshal(d, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
// d, err = yaml.Marshal(&config)
// if err != nil {
// log.Fatalf("error: %v", err)
// }
// fmt.Printf("--- m dump:\n%s\n\n", string(d))
return config, nil
}
// MakeRuns makes Run objects for each combination of Backend and Test
// in the config
func (c *Config) MakeRuns() (runs Runs) {
for _, backend := range c.Backends {
for _, test := range c.Tests {
runs = append(runs, backend.MakeRuns(&test)...)
}
}
return runs
}
// Filter the Backends with the remotes passed in.
//
// If no backend is found with a remote is found then synthesize one
func (c *Config) filterBackendsByRemotes(remotes []string) {
var newBackends []Backend
for _, name := range remotes {
found := false
for i := range c.Backends {
if c.Backends[i].Remote == name {
newBackends = append(newBackends, c.Backends[i])
found = true
}
}
if !found {
log.Printf("Remote %q not found - inserting with default flags", name)
// Lookup which backend
fsInfo, _, _, _, err := fs.ConfigFs(name)
if err != nil {
log.Fatalf("couldn't find remote %q: %v", name, err)
}
newBackends = append(newBackends, Backend{Backend: fsInfo.FileName(), Remote: name})
}
}
c.Backends = newBackends
}
// Filter the Backends with the backendNames passed in
func (c *Config) filterBackendsByBackends(backendNames []string) {
var newBackends []Backend
for _, name := range backendNames {
for i := range c.Backends {
if c.Backends[i].Backend == name {
newBackends = append(newBackends, c.Backends[i])
}
}
}
c.Backends = newBackends
}
// Filter the incoming tests into the backends selected
func (c *Config) filterTests(paths []string) {
var newTests []Test
for _, path := range paths {
for i := range c.Tests {
if c.Tests[i].Path == path {
newTests = append(newTests, c.Tests[i])
}
}
}
c.Tests = newTests
}