a7a8372976
Previously only the fs being checked on gets passed to GetModifyWindow(). However, in most tests, the test files are generated in the local fs and transferred to the remote fs. So the local fs time precision has to be taken into account. This meant that on Windows the time tests failed because the local fs has a time precision of 100ns. Checking remote items uploaded from local fs on Windows also requires a modify window of 100ns.
549 lines
15 KiB
Go
549 lines
15 KiB
Go
package operations_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/accounting"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/lib/readers"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
|
|
r := fstest.NewRun(t)
|
|
defer r.Finalise()
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
|
|
addBuffers := func(opt *operations.CheckOpt) {
|
|
opt.Combined = new(bytes.Buffer)
|
|
opt.MissingOnSrc = new(bytes.Buffer)
|
|
opt.MissingOnDst = new(bytes.Buffer)
|
|
opt.Match = new(bytes.Buffer)
|
|
opt.Differ = new(bytes.Buffer)
|
|
opt.Error = new(bytes.Buffer)
|
|
}
|
|
|
|
sortLines := func(in string) []string {
|
|
if in == "" {
|
|
return []string{}
|
|
}
|
|
lines := strings.Split(in, "\n")
|
|
sort.Strings(lines)
|
|
return lines
|
|
}
|
|
|
|
checkBuffer := func(name string, want map[string]string, out io.Writer) {
|
|
expected := want[name]
|
|
buf, ok := out.(*bytes.Buffer)
|
|
require.True(t, ok)
|
|
assert.Equal(t, sortLines(expected), sortLines(buf.String()), name)
|
|
}
|
|
|
|
checkBuffers := func(opt *operations.CheckOpt, want map[string]string) {
|
|
checkBuffer("combined", want, opt.Combined)
|
|
checkBuffer("missingonsrc", want, opt.MissingOnSrc)
|
|
checkBuffer("missingondst", want, opt.MissingOnDst)
|
|
checkBuffer("match", want, opt.Match)
|
|
checkBuffer("differ", want, opt.Differ)
|
|
checkBuffer("error", want, opt.Error)
|
|
}
|
|
|
|
check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) {
|
|
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
|
accounting.GlobalStats().ResetCounters()
|
|
var buf bytes.Buffer
|
|
log.SetOutput(&buf)
|
|
defer func() {
|
|
log.SetOutput(os.Stderr)
|
|
}()
|
|
opt := operations.CheckOpt{
|
|
Fdst: r.Fremote,
|
|
Fsrc: r.Flocal,
|
|
OneWay: oneway,
|
|
}
|
|
addBuffers(&opt)
|
|
err := checkFunction(ctx, &opt)
|
|
gotErrors := accounting.GlobalStats().GetErrors()
|
|
gotChecks := accounting.GlobalStats().GetChecks()
|
|
if wantErrors == 0 && err != nil {
|
|
t.Errorf("%d: Got error when not expecting one: %v", i, err)
|
|
}
|
|
if wantErrors != 0 && err == nil {
|
|
t.Errorf("%d: No error when expecting one", i)
|
|
}
|
|
if wantErrors != gotErrors {
|
|
t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
|
|
}
|
|
if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
|
|
t.Errorf("%d: Total files matching line missing", i)
|
|
}
|
|
if wantChecks != gotChecks {
|
|
t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
|
|
}
|
|
checkBuffers(&opt, wantOutput)
|
|
})
|
|
}
|
|
|
|
file1 := r.WriteBoth(ctx, "rutabaga", "is tasty", t3)
|
|
r.CheckRemoteItems(t, file1)
|
|
r.CheckLocalItems(t, file1)
|
|
check(1, 0, 1, false, map[string]string{
|
|
"combined": "= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
|
|
r.CheckLocalItems(t, file1, file2)
|
|
check(2, 1, 1, false, map[string]string{
|
|
"combined": "+ potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "potato2\n",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file3 := r.WriteObject(ctx, "empty space", "-", t2)
|
|
r.CheckRemoteItems(t, file1, file3)
|
|
check(3, 2, 1, false, map[string]string{
|
|
"combined": "- empty space\n+ potato2\n= rutabaga\n",
|
|
"missingonsrc": "empty space\n",
|
|
"missingondst": "potato2\n",
|
|
"match": "rutabaga\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2r := file2
|
|
if ci.SizeOnly {
|
|
file2r = r.WriteObject(ctx, "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
|
|
} else {
|
|
r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1)
|
|
}
|
|
r.CheckRemoteItems(t, file1, file2r, file3)
|
|
check(4, 1, 2, false, map[string]string{
|
|
"combined": "- empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "empty space\n",
|
|
"missingondst": "",
|
|
"match": "rutabaga\npotato2\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file3r := file3
|
|
file3l := r.WriteFile("empty space", "DIFFER", t2)
|
|
r.CheckLocalItems(t, file1, file2, file3l)
|
|
check(5, 1, 3, false, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
|
|
file4 := r.WriteObject(ctx, "remotepotato", "------------------------------------------------------------", t1)
|
|
r.CheckRemoteItems(t, file1, file2r, file3r, file4)
|
|
check(6, 2, 3, false, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
|
|
"missingonsrc": "remotepotato\n",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
check(7, 1, 3, true, map[string]string{
|
|
"combined": "* empty space\n= potato2\n= rutabaga\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato2\nrutabaga\n",
|
|
"differ": "empty space\n",
|
|
"error": "",
|
|
})
|
|
}
|
|
|
|
func TestCheck(t *testing.T) {
|
|
testCheck(t, operations.Check)
|
|
}
|
|
|
|
func TestCheckFsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
dstFs, err := fs.NewFs(ctx, "non-existent")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srcFs, err := fs.NewFs(ctx, "non-existent")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opt := operations.CheckOpt{
|
|
Fdst: dstFs,
|
|
Fsrc: srcFs,
|
|
OneWay: false,
|
|
}
|
|
err = operations.Check(ctx, &opt)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestCheckDownload(t *testing.T) {
|
|
testCheck(t, operations.CheckDownload)
|
|
}
|
|
|
|
func TestCheckSizeOnly(t *testing.T) {
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
ci.SizeOnly = true
|
|
defer func() { ci.SizeOnly = false }()
|
|
TestCheck(t)
|
|
}
|
|
|
|
func TestCheckEqualReaders(t *testing.T) {
|
|
b65a := make([]byte, 65*1024)
|
|
b65b := make([]byte, 65*1024)
|
|
b65b[len(b65b)-1] = 1
|
|
b66 := make([]byte, 66*1024)
|
|
|
|
differ, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, differ, false)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
myErr := errors.New("sentinel")
|
|
wrap := func(b []byte) io.Reader {
|
|
r := bytes.NewBuffer(b)
|
|
e := readers.ErrorReader{Err: myErr}
|
|
return io.MultiReader(r, e)
|
|
}
|
|
|
|
differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
|
|
differ, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a))
|
|
assert.Equal(t, myErr, err)
|
|
assert.Equal(t, differ, true)
|
|
}
|
|
|
|
func TestParseSumFile(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
defer r.Finalise()
|
|
ctx := context.Background()
|
|
|
|
const sumFile = "test.sum"
|
|
|
|
samples := []struct {
|
|
hash, sep, name string
|
|
ok bool
|
|
}{
|
|
{"1", " ", "file1", true},
|
|
{"2", " *", "file2", true},
|
|
{"3", " ", " file3 ", true},
|
|
{"4", " ", "\tfile3\t", true},
|
|
{"5", " ", "file5", false},
|
|
{"6", "\t", "file6", false},
|
|
{"7", " \t", " file7 ", false},
|
|
{"", " ", "file8", false},
|
|
{"", "", "file9", false},
|
|
}
|
|
|
|
for _, eol := range []string{"\n", "\r\n"} {
|
|
data := &bytes.Buffer{}
|
|
wantNum := 0
|
|
for _, s := range samples {
|
|
_, _ = data.WriteString(s.hash + s.sep + s.name + eol)
|
|
if s.ok {
|
|
wantNum++
|
|
}
|
|
}
|
|
|
|
_ = r.WriteObject(ctx, sumFile, data.String(), t1)
|
|
file, err := r.Fremote.NewObject(ctx, sumFile)
|
|
assert.NoError(t, err)
|
|
sums, err := operations.ParseSumFile(ctx, file)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, wantNum, len(sums))
|
|
for _, s := range samples {
|
|
if s.ok {
|
|
assert.Equal(t, s.hash, sums[s.name])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func testCheckSum(t *testing.T, download bool) {
|
|
const dataDir = "data"
|
|
const sumFile = "test.sum"
|
|
|
|
hashType := hash.MD5
|
|
const (
|
|
testString1 = "Hello, World!"
|
|
testDigest1 = "65a8e27d8879283831b664bd8b7f0ad4"
|
|
testDigest1Upper = "65A8E27D8879283831B664BD8B7F0AD4"
|
|
testString2 = "I am the walrus"
|
|
testDigest2 = "87396e030ef3f5b35bbf85c0a09a4fb3"
|
|
testDigest2Mixed = "87396e030EF3f5b35BBf85c0a09a4FB3"
|
|
)
|
|
|
|
type wantType map[string]string
|
|
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
defer r.Finalise()
|
|
|
|
subRemote := r.FremoteName
|
|
if !strings.HasSuffix(subRemote, ":") {
|
|
subRemote += "/"
|
|
}
|
|
subRemote += dataDir
|
|
dataFs, err := fs.NewFs(ctx, subRemote)
|
|
require.NoError(t, err)
|
|
|
|
if !download && !dataFs.Hashes().Contains(hashType) {
|
|
t.Skipf("%s lacks %s, skipping", dataFs, hashType)
|
|
}
|
|
|
|
makeFile := func(name, content string) fstest.Item {
|
|
remote := dataDir + "/" + name
|
|
return r.WriteObject(ctx, remote, content, t1)
|
|
}
|
|
|
|
makeSums := func(sums operations.HashSums) fstest.Item {
|
|
files := make([]string, 0, len(sums))
|
|
for name := range sums {
|
|
files = append(files, name)
|
|
}
|
|
sort.Strings(files)
|
|
buf := &bytes.Buffer{}
|
|
for _, name := range files {
|
|
_, _ = fmt.Fprintf(buf, "%s %s\n", sums[name], name)
|
|
}
|
|
return r.WriteObject(ctx, sumFile, buf.String(), t1)
|
|
}
|
|
|
|
sortLines := func(in string) []string {
|
|
if in == "" {
|
|
return []string{}
|
|
}
|
|
lines := strings.Split(in, "\n")
|
|
sort.Strings(lines)
|
|
return lines
|
|
}
|
|
|
|
checkResult := func(runNo int, want wantType, name string, out io.Writer) {
|
|
expected := want[name]
|
|
buf, ok := out.(*bytes.Buffer)
|
|
require.True(t, ok)
|
|
assert.Equal(t, sortLines(expected), sortLines(buf.String()), "wrong %s result in run %d", name, runNo)
|
|
}
|
|
|
|
checkRun := func(runNo, wantChecks, wantErrors int, want wantType) {
|
|
accounting.GlobalStats().ResetCounters()
|
|
buf := new(bytes.Buffer)
|
|
log.SetOutput(buf)
|
|
defer log.SetOutput(os.Stderr)
|
|
|
|
opt := operations.CheckOpt{
|
|
Combined: new(bytes.Buffer),
|
|
Match: new(bytes.Buffer),
|
|
Differ: new(bytes.Buffer),
|
|
Error: new(bytes.Buffer),
|
|
MissingOnSrc: new(bytes.Buffer),
|
|
MissingOnDst: new(bytes.Buffer),
|
|
}
|
|
err := operations.CheckSum(ctx, dataFs, r.Fremote, sumFile, hashType, &opt, download)
|
|
|
|
gotErrors := int(accounting.GlobalStats().GetErrors())
|
|
if wantErrors == 0 {
|
|
assert.NoError(t, err, "unexpected error in run %d", runNo)
|
|
}
|
|
if wantErrors > 0 {
|
|
assert.Error(t, err, "no expected error in run %d", runNo)
|
|
}
|
|
assert.Equal(t, wantErrors, gotErrors, "wrong error count in run %d", runNo)
|
|
|
|
gotChecks := int(accounting.GlobalStats().GetChecks())
|
|
if wantChecks > 0 || gotChecks > 0 {
|
|
assert.Contains(t, buf.String(), "matching files", "missing matching files in run %d", runNo)
|
|
}
|
|
assert.Equal(t, wantChecks, gotChecks, "wrong number of checks in run %d", runNo)
|
|
|
|
checkResult(runNo, want, "combined", opt.Combined)
|
|
checkResult(runNo, want, "missingonsrc", opt.MissingOnSrc)
|
|
checkResult(runNo, want, "missingondst", opt.MissingOnDst)
|
|
checkResult(runNo, want, "match", opt.Match)
|
|
checkResult(runNo, want, "differ", opt.Differ)
|
|
checkResult(runNo, want, "error", opt.Error)
|
|
}
|
|
|
|
check := func(runNo, wantChecks, wantErrors int, wantResults wantType) {
|
|
runName := fmt.Sprintf("subtest%d", runNo)
|
|
t.Run(runName, func(t *testing.T) {
|
|
checkRun(runNo, wantChecks, wantErrors, wantResults)
|
|
})
|
|
}
|
|
|
|
file1 := makeFile("banana", testString1)
|
|
fcsums := makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1)
|
|
check(1, 1, 0, wantType{
|
|
"combined": "= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
file2 := makeFile("potato", testString2)
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(2, 2, 1, wantType{
|
|
"combined": "- potato\n= banana\n",
|
|
"missingonsrc": "potato\n",
|
|
"missingondst": "",
|
|
"match": "banana\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(3, 2, 0, wantType{
|
|
"combined": "= potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest2,
|
|
"potato": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(4, 2, 1, wantType{
|
|
"combined": "* banana\n= potato\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "potato\n",
|
|
"differ": "banana\n",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest2,
|
|
"orange": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(5, 2, 1, wantType{
|
|
"combined": "+ orange\n= potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "orange\n",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1,
|
|
"potato": testDigest1,
|
|
"orange": testDigest2,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(6, 2, 2, wantType{
|
|
"combined": "+ orange\n* potato\n= banana\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "orange\n",
|
|
"match": "banana\n",
|
|
"differ": "potato\n",
|
|
"error": "",
|
|
})
|
|
|
|
// test mixed-case checksums
|
|
file1 = makeFile("banana", testString1)
|
|
file2 = makeFile("potato", testString2)
|
|
fcsums = makeSums(operations.HashSums{
|
|
"banana": testDigest1Upper,
|
|
"potato": testDigest2Mixed,
|
|
})
|
|
r.CheckRemoteItems(t, fcsums, file1, file2)
|
|
check(7, 2, 0, wantType{
|
|
"combined": "= banana\n= potato\n",
|
|
"missingonsrc": "",
|
|
"missingondst": "",
|
|
"match": "banana\npotato\n",
|
|
"differ": "",
|
|
"error": "",
|
|
})
|
|
}
|
|
|
|
func TestCheckSum(t *testing.T) {
|
|
testCheckSum(t, false)
|
|
}
|
|
|
|
func TestCheckSumDownload(t *testing.T) {
|
|
testCheckSum(t, true)
|
|
}
|