rclone/fs/operations/copy_test.go
nielash 87ec26001f local: add server-side copy with xattrs on macOS (part-fix #1710)
Before this change, macOS-specific metadata was not preserved by rclone, even for
local-to-local transfers (it does not use the "user." prefix, nor is Mac metadata
limited to xattrs.) Additionally, rclone did not take advantage of APFS's native
"cloning" functionality for fast and deduplicated transfers.

After this change, local (on macOS only) supports "server-side copy" similarly to
other remotes, and achieves this by using (when possible) macOS's native APFS
"cloning", which is the same underlying mechanism deployed when a user
duplicates a file via the Finder UI. This has several advantages over the
previous behavior:

- It is extremely fast (even large files can be cloned instantly)
- It is very efficient in terms of storage, as it automatically deduplicates when
possible (i.e. so that having two identical files does not consume more storage
than having just one.) (The concept is similar to a "hard link", but subsequent
modifications will not affect the original file.)
- It preserves Mac-specific metadata to the maximum degree, including not only
xattrs but also metadata not easily settable by other methods, including Finder
and Spotlight params.

When server-side "clone" is not available (for example, on non-APFS volumes), it
falls back to server-side "copy" (still preserving metadata but using more disk
storage.) It is only used when both remotes are local (and not wrapped by other
remotes, such as crypt.) The behavior of local on non-mac systems is unchanged.
2024-08-15 15:36:38 +01:00

507 lines
14 KiB
Go

package operations_test
import (
"context"
"crypto/rand"
"errors"
"fmt"
"os"
"path"
"runtime"
"sort"
"strings"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTruncateString(t *testing.T) {
for _, test := range []struct {
in string
n int
want string
}{
{
in: "",
n: 0,
want: "",
}, {
in: "Hello World",
n: 5,
want: "Hello",
}, {
in: "ááááá",
n: 5,
want: "áá",
}, {
in: "ááááá\xFF\xFF",
n: 5,
want: "áá\xc3",
}, {
in: "世世世世世",
n: 7,
want: "世世",
}, {
in: "🙂🙂🙂🙂🙂",
n: 16,
want: "🙂🙂🙂🙂",
}, {
in: "🙂🙂🙂🙂🙂",
n: 15,
want: "🙂🙂🙂",
}, {
in: "🙂🙂🙂🙂🙂",
n: 14,
want: "🙂🙂🙂",
}, {
in: "🙂🙂🙂🙂🙂",
n: 13,
want: "🙂🙂🙂",
}, {
in: "🙂🙂🙂🙂🙂",
n: 12,
want: "🙂🙂🙂",
}, {
in: "🙂🙂🙂🙂🙂",
n: 11,
want: "🙂🙂",
}, {
in: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
n: 100,
want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
}, {
in: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
n: 100,
want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
}, {
in: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
n: 100,
want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
}, {
in: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
n: 100,
want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
}, {
in: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
n: 100,
want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ",
},
} {
got := operations.TruncateString(test.in, test.n)
assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in))
assert.LessOrEqual(t, len(got), test.n)
assert.GreaterOrEqual(t, len(got), test.n-3)
}
}
func TestCopyFile(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("file1", "file1 contents", t1)
r.CheckLocalItems(t, file1)
file2 := file1
file2.Path = "sub/file2"
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
}
// Find the longest file name for writing to local
func maxLengthFileName(t *testing.T, r *fstest.Run) string {
require.NoError(t, r.Flocal.Mkdir(context.Background(), "")) // create the root
const maxLen = 16 * 1024
name := strings.Repeat("A", maxLen)
i := sort.Search(len(name), func(i int) (fail bool) {
filePath := path.Join(r.LocalName, name[:i])
err := os.WriteFile(filePath, []byte{0}, 0777)
if err != nil {
return true
}
err = os.Remove(filePath)
if err != nil {
t.Logf("Failed to remove test file: %v", err)
}
return false
})
return name[:i-1]
}
// Check we can copy a file of maximum name length
func TestCopyLongFile(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
if !r.Fremote.Features().IsLocal {
t.Skip("Test only runs on local")
}
// Find the maximum length of file we can write
name := maxLengthFileName(t, r)
t.Logf("Max length of file name is %d", len(name))
file1 := r.WriteFile(name, "file1 contents", t1)
r.CheckLocalItems(t, file1)
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
}
func TestCopyFileBackupDir(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
if !operations.CanServerSideMove(r.Fremote) {
t.Skip("Skipping test as remote does not support server-side move or copy")
}
ci.BackupDir = r.FremoteName + "/backup"
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
r.CheckLocalItems(t, file1)
file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1)
r.CheckRemoteItems(t, file1old)
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
file1old.Path = "backup/dst/file1"
r.CheckRemoteItems(t, file1old, file1)
}
// Test with CompareDest set
func TestCopyFileCompareDest(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
require.NoError(t, err)
// check empty dest, empty compare
file1 := r.WriteFile("one", "one", t1)
r.CheckLocalItems(t, file1)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
r.CheckRemoteItems(t, file1dst)
// check old dest, empty compare
file1b := r.WriteFile("one", "onet2", t2)
r.CheckRemoteItems(t, file1dst)
r.CheckLocalItems(t, file1b)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
r.CheckRemoteItems(t, file1bdst)
// check old dest, new compare
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
r.CheckRemoteItems(t, file2, file3)
r.CheckLocalItems(t, file1c)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
r.CheckRemoteItems(t, file2, file3)
// check empty dest, new compare
file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
r.CheckRemoteItems(t, file2, file3, file4)
r.CheckLocalItems(t, file1c, file5)
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
r.CheckRemoteItems(t, file2, file3, file4)
// check new dest, new compare
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
r.CheckRemoteItems(t, file2, file3, file4)
// check empty dest, old compare
file5b := r.WriteFile("two", "twot3", t3)
r.CheckRemoteItems(t, file2, file3, file4)
r.CheckLocalItems(t, file1c, file5b)
err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path)
require.NoError(t, err)
file5bdst := file5b
file5bdst.Path = "dst/two"
r.CheckRemoteItems(t, file2, file3, file4, file5bdst)
}
// Test with CopyDest set
func TestCopyFileCopyDest(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
if r.Fremote.Features().Copy == nil {
t.Skip("Skipping test as remote does not support server-side copy")
}
ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
require.NoError(t, err)
// check empty dest, empty copy
file1 := r.WriteFile("one", "one", t1)
r.CheckLocalItems(t, file1)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
r.CheckRemoteItems(t, file1dst)
// check old dest, empty copy
file1b := r.WriteFile("one", "onet2", t2)
r.CheckRemoteItems(t, file1dst)
r.CheckLocalItems(t, file1b)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
r.CheckRemoteItems(t, file1bdst)
// check old dest, new copy, backup-dir
ci.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
r.CheckRemoteItems(t, file2, file3)
r.CheckLocalItems(t, file1c)
err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
file2dst := file2
file2dst.Path = "dst/one"
file3.Path = "BackupDir/one"
r.CheckRemoteItems(t, file2, file2dst, file3)
ci.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
r.CheckRemoteItems(t, file2, file2dst, file3, file4)
r.CheckLocalItems(t, file1c, file5)
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
file4dst := file4
file4dst.Path = "dst/two"
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
// check new dest, new copy
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
// check empty dest, old copy
file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2)
file7 := r.WriteFile("three", "threet3", t3)
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6)
r.CheckLocalItems(t, file1c, file5, file7)
err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path)
require.NoError(t, err)
file7dst := file7
file7dst.Path = "dst/three"
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}
func TestCopyInplace(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
if !r.Fremote.Features().PartialUploads {
t.Skip("Partial uploads not supported")
}
ci.Inplace = true
file1 := r.WriteFile("file1", "file1 contents", t1)
r.CheckLocalItems(t, file1)
file2 := file1
file2.Path = "sub/file2"
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
}
func TestCopyLongFileName(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
if !r.Fremote.Features().PartialUploads {
t.Skip("Partial uploads not supported")
}
ci.Inplace = false // the default
file1 := r.WriteFile("file1", "file1 contents", t1)
r.CheckLocalItems(t, file1)
file2 := file1
file2.Path = "sub/" + strings.Repeat("file2", 30)
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file2)
}
func TestCopyFileMaxTransfer(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
defer accounting.Stats(ctx).ResetCounters()
const sizeCutoff = 2048
// Make random incompressible data
randomData := make([]byte, sizeCutoff)
_, err := rand.Read(randomData)
require.NoError(t, err)
randomString := string(randomData)
file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1)
file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2)
file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2)
file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2)
// Cutoff mode: Hard
ci.MaxTransfer = sizeCutoff
ci.CutoffMode = fs.CutoffModeHard
if runtime.GOOS == "darwin" {
// disable server-side copies as they don't count towards transfer size stats
r.Flocal.Features().Disable("Copy")
if r.Fremote.Features().IsLocal {
r.Fremote.Features().Disable("Copy")
}
}
// file1: Show a small file gets transferred OK
accounting.Stats(ctx).ResetCounters()
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1, file2, file3, file4)
r.CheckRemoteItems(t, file1)
// file2: show a large file does not get transferred
accounting.Stats(ctx).ResetCounters()
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path)
require.NotNil(t, err, "Did not get expected max transfer limit error")
if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) {
t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal")
// Sometimes the backends or their SDKs don't pass the
// error through properly, so check that it at least
// has the text we expect in.
assert.Contains(t, err.Error(), "max transfer limit reached")
}
r.CheckLocalItems(t, file1, file2, file3, file4)
r.CheckRemoteItems(t, file1)
// Cutoff mode: Cautious
ci.CutoffMode = fs.CutoffModeCautious
// file3: show a large file does not get transferred
accounting.Stats(ctx).ResetCounters()
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path)
require.NotNil(t, err)
assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful))
r.CheckLocalItems(t, file1, file2, file3, file4)
r.CheckRemoteItems(t, file1)
if isChunker(r.Fremote) {
t.Log("skipping remainder of test for chunker as it involves multiple transfers")
return
}
// Cutoff mode: Soft
ci.CutoffMode = fs.CutoffModeSoft
// file4: show a large file does get transferred this time
accounting.Stats(ctx).ResetCounters()
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path)
require.NoError(t, err)
r.CheckLocalItems(t, file1, file2, file3, file4)
r.CheckRemoteItems(t, file1, file4)
}