8e507075d1
Reference: https://pkg.go.dev/testing#T.Cleanup Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
280 lines
9.2 KiB
Go
280 lines
9.2 KiB
Go
package operations_test
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fs/walk"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/lib/random"
|
|
"github.com/spf13/pflag"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Check flag satisfies the interface
|
|
var _ pflag.Value = (*operations.DeduplicateMode)(nil)
|
|
|
|
func skipIfCantDedupe(t *testing.T, f fs.Fs) {
|
|
if !f.Features().DuplicateFiles {
|
|
t.Skip("Can't test deduplicate - no duplicate files possible")
|
|
}
|
|
if f.Features().PutUnchecked == nil {
|
|
t.Skip("Can't test deduplicate - no PutUnchecked")
|
|
}
|
|
if f.Features().MergeDirs == nil {
|
|
t.Skip("Can't test deduplicate - no MergeDirs")
|
|
}
|
|
}
|
|
|
|
func skipIfNoHash(t *testing.T, f fs.Fs) {
|
|
if f.Hashes().GetOne() == hash.None {
|
|
t.Skip("Can't run this test without a hash")
|
|
}
|
|
}
|
|
|
|
func skipIfNoModTime(t *testing.T, f fs.Fs) {
|
|
if f.Precision() >= fs.ModTimeNotSupported {
|
|
t.Skip("Can't run this test without modtimes")
|
|
}
|
|
}
|
|
|
|
func TestDeduplicateInteractive(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
skipIfNoHash(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file1)
|
|
}
|
|
|
|
func TestDeduplicateSkip(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
haveHash := r.Fremote.Hashes().GetOne() != hash.None
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
files := []fstest.Item{file1}
|
|
if haveHash {
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
files = append(files, file2)
|
|
}
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
|
|
files = append(files, file3)
|
|
r.CheckWithDuplicates(t, files...)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckWithDuplicates(t, file1, file3)
|
|
}
|
|
|
|
func TestDeduplicateSizeOnly(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
ctx := context.Background()
|
|
ci := fs.GetConfig(ctx)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "THIS IS ONE", t1)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
ci.SizeOnly = true
|
|
defer func() {
|
|
ci.SizeOnly = false
|
|
}()
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckWithDuplicates(t, file1, file3)
|
|
}
|
|
|
|
func TestDeduplicateFirst(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst, false)
|
|
require.NoError(t, err)
|
|
|
|
// list until we get one object
|
|
var objects, size int64
|
|
for try := 1; try <= *fstest.ListRetries; try++ {
|
|
objects, size, _, err = operations.Count(context.Background(), r.Fremote)
|
|
require.NoError(t, err)
|
|
if objects == 1 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
assert.Equal(t, int64(1), objects)
|
|
if size != file1.Size && size != file2.Size && size != file3.Size {
|
|
t.Errorf("Size not one of the object sizes %d", size)
|
|
}
|
|
}
|
|
|
|
func TestDeduplicateNewest(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
skipIfNoModTime(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file3)
|
|
}
|
|
|
|
func TestDeduplicateNewestByHash(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfNoHash(t, r.Fremote)
|
|
skipIfNoModTime(t, r.Fremote)
|
|
contents := random.String(100)
|
|
|
|
file1 := r.WriteObject(context.Background(), "one", contents, t1)
|
|
file2 := r.WriteObject(context.Background(), "also/one", contents, t2)
|
|
file3 := r.WriteObject(context.Background(), "another", contents, t3)
|
|
file4 := r.WriteObject(context.Background(), "not-one", "stuff", t3)
|
|
r.CheckRemoteItems(t, file1, file2, file3, file4)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, true)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file3, file4)
|
|
}
|
|
|
|
func TestDeduplicateOldest(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file1)
|
|
}
|
|
|
|
func TestDeduplicateLargest(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file3)
|
|
}
|
|
|
|
func TestDeduplicateSmallest(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
|
|
r.CheckWithDuplicates(t, file1, file2, file3)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSmallest, false)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckRemoteItems(t, file1)
|
|
}
|
|
|
|
func TestDeduplicateRename(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
skipIfCantDedupe(t, r.Fremote)
|
|
|
|
file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
|
|
file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2)
|
|
file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3)
|
|
file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1)
|
|
r.CheckWithDuplicates(t, file1, file2, file3, file4)
|
|
|
|
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename, false)
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
|
|
entries.ForObject(func(o fs.Object) {
|
|
remote := o.Remote()
|
|
if remote != "one-1.txt" &&
|
|
remote != "one-2.txt" &&
|
|
remote != "one-3.txt" &&
|
|
remote != "one-4.txt" {
|
|
t.Errorf("Bad file name after rename %q", remote)
|
|
}
|
|
size := o.Size()
|
|
if size != file1.Size &&
|
|
size != file2.Size &&
|
|
size != file3.Size &&
|
|
size != file4.Size {
|
|
t.Errorf("Size not one of the object sizes %d", size)
|
|
}
|
|
if remote == "one-1.txt" && size != file4.Size {
|
|
t.Errorf("Existing non-duplicate file modified %q", remote)
|
|
}
|
|
})
|
|
return nil
|
|
}))
|
|
}
|
|
|
|
// This should really be a unit test, but the test framework there
|
|
// doesn't have enough tools to make it easy
|
|
func TestMergeDirs(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
|
|
mergeDirs := r.Fremote.Features().MergeDirs
|
|
if mergeDirs == nil {
|
|
t.Skip("Can't merge directories")
|
|
}
|
|
|
|
file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1)
|
|
file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2)
|
|
file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3)
|
|
|
|
objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 3, len(dirs))
|
|
assert.Equal(t, 0, len(objs))
|
|
|
|
err = mergeDirs(context.Background(), dirs)
|
|
require.NoError(t, err)
|
|
|
|
file2.Path = "dupe1/two.txt"
|
|
file3.Path = "dupe1/three.txt"
|
|
r.CheckRemoteItems(t, file1, file2, file3)
|
|
|
|
objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, len(dirs))
|
|
assert.Equal(t, 0, len(objs))
|
|
assert.Equal(t, "dupe1", dirs[0].Remote())
|
|
}
|