2016-07-04 12:12:33 +00:00
// Test sync/copy/move
2018-01-12 16:30:54 +00:00
package sync
2016-07-04 12:12:33 +00:00
import (
2023-10-01 09:02:56 +00:00
"bytes"
2019-06-17 08:34:30 +00:00
"context"
2021-11-04 10:12:57 +00:00
"errors"
2019-07-25 10:28:27 +00:00
"fmt"
2024-02-06 16:00:34 +00:00
"io"
2023-10-01 09:02:56 +00:00
"os"
"os/exec"
2017-09-17 13:05:33 +00:00
"runtime"
2023-10-01 09:02:56 +00:00
"sort"
2019-09-05 20:29:35 +00:00
"strings"
2016-07-04 12:12:33 +00:00
"testing"
"time"
2023-10-01 09:02:56 +00:00
mutex "sync" // renamed as "sync" already in use
2019-07-28 17:47:38 +00:00
_ "github.com/rclone/rclone/backend/all" // import all backends
2024-02-29 00:29:38 +00:00
"github.com/rclone/rclone/cmd/bisync/bilib"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
2016-07-04 12:12:33 +00:00
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
2017-09-08 15:19:41 +00:00
"golang.org/x/text/unicode/norm"
2016-07-04 12:12:33 +00:00
)
2018-01-12 16:30:54 +00:00
// Some times used in the tests
var (
t1 = fstest . Time ( "2001-02-03T04:05:06.499999999Z" )
t2 = fstest . Time ( "2011-12-25T12:59:59.123456789Z" )
t3 = fstest . Time ( "2011-12-30T12:59:59.000000000Z" )
)
// TestMain drives the tests
func TestMain ( m * testing . M ) {
fstest . TestMain ( m )
}
2016-07-04 12:12:33 +00:00
// Check dry run is working
func TestCopyWithDryRun ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2016-07-04 12:12:33 +00:00
2020-11-05 11:33:32 +00:00
ci . DryRun = true
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t ) // error expected here because dry-run
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t )
2016-07-04 12:12:33 +00:00
}
// Now without dry run
func TestCopy ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2024-02-06 16:00:34 +00:00
_ , err := operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir" , t2 )
if err != nil && ! errors . Is ( err , fs . ErrorNotImplemented ) {
require . NoError ( t , err )
}
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2024-02-06 16:00:34 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2024-02-06 16:00:34 +00:00
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , "sub dir" )
}
2024-04-04 17:03:20 +00:00
func testCopyMetadata ( t * testing . T , createEmptySrcDirs bool ) {
2024-02-06 16:00:34 +00:00
ctx := context . Background ( )
ctx , ci := fs . AddConfig ( ctx )
ci . Metadata = true
r := fstest . NewRun ( t )
features := r . Fremote . Features ( )
if ! features . ReadMetadata && ! features . WriteMetadata && ! features . UserMetadata &&
! features . ReadDirMetadata && ! features . WriteDirMetadata && ! features . UserDirMetadata {
t . Skip ( "Skipping as metadata not supported" )
}
2024-06-08 09:40:42 +00:00
if createEmptySrcDirs && ! features . CanHaveEmptyDirectories {
t . Skip ( "Skipping as can't have empty directories" )
}
2024-02-06 16:00:34 +00:00
const content = "hello metadata world!"
const dirPath = "metadata sub dir"
2024-04-04 17:03:20 +00:00
const emptyDirPath = "empty metadata sub dir"
2024-02-06 16:00:34 +00:00
const filePath = dirPath + "/hello metadata world"
fileMetadata := fs . Metadata {
// System metadata supported by all backends
"mtime" : t1 . Format ( time . RFC3339Nano ) ,
// User metadata
"potato" : "jersey" ,
}
dirMetadata := fs . Metadata {
// System metadata supported by all backends
"mtime" : t2 . Format ( time . RFC3339Nano ) ,
// User metadata
"potato" : "king edward" ,
}
// Make the directory with metadata - may fall back to Mkdir
_ , err := operations . MkdirMetadata ( ctx , r . Flocal , dirPath , dirMetadata )
require . NoError ( t , err )
2024-04-04 17:03:20 +00:00
// Make the empty directory with metadata - may fall back to Mkdir
_ , err = operations . MkdirMetadata ( ctx , r . Flocal , emptyDirPath , dirMetadata )
require . NoError ( t , err )
2024-02-06 16:00:34 +00:00
// Upload the file with metadata
in := io . NopCloser ( bytes . NewBufferString ( content ) )
_ , err = operations . Rcat ( ctx , r . Flocal , filePath , in , t1 , fileMetadata )
require . NoError ( t , err )
file1 := fstest . NewItem ( filePath , content , t1 )
// Reset the time of the directory
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , dirPath , t2 )
if err != nil && ! errors . Is ( err , fs . ErrorNotImplemented ) {
require . NoError ( t , err )
}
ctx = predictDstFromLogger ( ctx )
2024-04-04 17:03:20 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , createEmptySrcDirs )
2024-02-06 16:00:34 +00:00
require . NoError ( t , err )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , dirPath )
// Check that the metadata on the directory and file is correct
2024-06-11 10:30:32 +00:00
if features . WriteMetadata && features . ReadMetadata {
2024-02-06 16:00:34 +00:00
fstest . CheckEntryMetadata ( ctx , t , r . Fremote , fstest . NewObject ( ctx , t , r . Fremote , filePath ) , fileMetadata )
}
2024-06-11 10:30:32 +00:00
if features . WriteDirMetadata && features . ReadDirMetadata {
2024-02-06 16:00:34 +00:00
fstest . CheckEntryMetadata ( ctx , t , r . Fremote , fstest . NewDirectory ( ctx , t , r . Fremote , dirPath ) , dirMetadata )
}
2024-04-04 17:03:20 +00:00
if ! createEmptySrcDirs {
// dir must not exist
_ , err := fstest . NewDirectoryRetries ( ctx , t , r . Fremote , emptyDirPath , 1 )
assert . Error ( t , err , "Not expecting to find empty directory" )
assert . True ( t , errors . Is ( err , fs . ErrorDirNotFound ) , fmt . Sprintf ( "expecting wrapped %#v not: %#v" , fs . ErrorDirNotFound , err ) )
} else {
// dir must exist
dir := fstest . NewDirectory ( ctx , t , r . Fremote , emptyDirPath )
if features . ReadDirMetadata {
fstest . CheckEntryMetadata ( ctx , t , r . Fremote , dir , dirMetadata )
}
}
}
func TestCopyMetadata ( t * testing . T ) {
testCopyMetadata ( t , true )
}
func TestCopyMetadataNoEmptyDirs ( t * testing . T ) {
testCopyMetadata ( t , false )
2016-07-04 12:12:33 +00:00
}
2019-06-20 11:50:25 +00:00
func TestCopyMissingDirectory ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2019-06-20 11:50:25 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2019-06-20 11:50:25 +00:00
2020-11-05 16:27:01 +00:00
nonExistingFs , err := fs . NewFs ( ctx , "/non-existing" )
2019-06-20 11:50:25 +00:00
if err != nil {
t . Fatal ( err )
}
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = CopyDir ( ctx , r . Fremote , nonExistingFs , false )
2019-06-20 11:50:25 +00:00
require . Error ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2019-06-20 11:50:25 +00:00
}
2018-11-25 16:49:38 +00:00
// Now with --no-traverse
func TestCopyNoTraverse ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2018-11-25 16:49:38 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . NoTraverse = true
2018-11-25 16:49:38 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Fremote , r . Flocal , false )
2018-11-25 16:49:38 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-11-25 16:49:38 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2018-11-25 16:49:38 +00:00
}
2020-05-15 10:39:07 +00:00
// Now with --check-first
func TestCopyCheckFirst ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-05-15 10:39:07 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . CheckFirst = true
2020-05-15 10:39:07 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Fremote , r . Flocal , false )
2020-05-15 10:39:07 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2020-05-15 10:39:07 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2020-05-15 10:39:07 +00:00
}
2018-11-25 16:49:38 +00:00
// Now with --no-traverse
func TestSyncNoTraverse ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2018-11-25 16:49:38 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . NoTraverse = true
2018-11-25 16:49:38 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2018-11-25 16:49:38 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-11-25 16:49:38 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2018-11-25 16:49:38 +00:00
}
2016-07-04 12:12:33 +00:00
// Test copy with depth
func TestCopyWithDepth ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
file2 := r . WriteFile ( "hello world2" , "hello world2" , t2 )
// Check the MaxDepth too
2020-11-05 11:33:32 +00:00
ci . MaxDepth = 1
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file2 )
2016-07-04 12:12:33 +00:00
}
2018-10-19 16:41:14 +00:00
// Test copy with files from
2019-02-13 17:14:51 +00:00
func testCopyWithFilesFrom ( t * testing . T , noTraverse bool ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2018-10-19 16:41:14 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "potato2" , "hello world" , t1 )
file2 := r . WriteFile ( "hello world2" , "hello world2" , t2 )
// Set the --files-from equivalent
f , err := filter . NewFilter ( nil )
require . NoError ( t , err )
require . NoError ( t , f . AddFile ( "potato2" ) )
require . NoError ( t , f . AddFile ( "notfound" ) )
2020-11-26 17:10:41 +00:00
// Change the active filter
ctx = filter . ReplaceConfig ( ctx , f )
2020-11-05 11:33:32 +00:00
ci . NoTraverse = noTraverse
2018-10-19 16:41:14 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , false )
2018-10-19 16:41:14 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-10-19 16:41:14 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file1 )
2018-10-19 16:41:14 +00:00
}
2019-02-13 17:14:51 +00:00
func TestCopyWithFilesFrom ( t * testing . T ) { testCopyWithFilesFrom ( t , false ) }
func TestCopyWithFilesFromAndNoTraverse ( t * testing . T ) { testCopyWithFilesFrom ( t , true ) }
2018-10-19 16:41:14 +00:00
2018-06-03 08:21:25 +00:00
// Test copy empty directories
func TestCopyEmptyDirectories ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2018-06-03 08:21:25 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2024-04-17 15:55:17 +00:00
_ , err := operations . MkdirModTime ( ctx , r . Flocal , "sub dir2/sub sub dir2" , t2 )
require . NoError ( t , err )
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir2" , t2 )
2018-06-03 08:21:25 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2018-06-03 08:21:25 +00:00
2024-02-06 16:00:34 +00:00
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir" , t1 )
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , true )
2019-03-06 08:43:46 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2019-03-06 08:43:46 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2019-03-06 08:43:46 +00:00
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
"sub dir2" ,
2024-04-17 15:55:17 +00:00
"sub dir2/sub sub dir2" ,
2019-03-06 08:43:46 +00:00
} ,
)
2024-02-06 16:00:34 +00:00
// Check that the modtimes of the directories are as expected
2024-04-17 15:55:17 +00:00
r . CheckDirectoryModTimes ( t , "sub dir" , "sub dir2" , "sub dir2/sub sub dir2" )
2019-03-06 08:43:46 +00:00
}
2024-04-04 17:03:20 +00:00
// Test copy empty directories when we are configured not to create them
func TestCopyNoEmptyDirectories ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
err := operations . Mkdir ( ctx , r . Flocal , "sub dir2" )
require . NoError ( t , err )
2024-04-17 15:55:17 +00:00
_ , err = operations . MkdirModTime ( ctx , r . Flocal , "sub dir2/sub sub dir2" , t2 )
require . NoError ( t , err )
2024-04-04 17:03:20 +00:00
r . Mkdir ( ctx , r . Fremote )
err = CopyDir ( ctx , r . Fremote , r . Flocal , false )
require . NoError ( t , err )
r . CheckRemoteListing (
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
} ,
)
}
2019-03-06 08:43:46 +00:00
// Test move empty directories
func TestMoveEmptyDirectories ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2019-03-06 08:43:46 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2024-02-06 16:00:34 +00:00
_ , err := operations . MkdirModTime ( ctx , r . Flocal , "sub dir2" , t2 )
2019-03-06 08:43:46 +00:00
require . NoError ( t , err )
2024-02-06 16:00:34 +00:00
subDir := fstest . NewDirectory ( ctx , t , r . Flocal , "sub dir" )
subDirT := subDir . ModTime ( ctx )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2019-03-06 08:43:46 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = MoveDir ( ctx , r . Fremote , r . Flocal , false , true )
2019-03-06 08:43:46 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2019-03-06 08:43:46 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2019-03-06 08:43:46 +00:00
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
"sub dir2" ,
} ,
)
2024-02-06 16:00:34 +00:00
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , "sub dir2" )
// Note that "sub dir" mod time is updated when file1 is deleted from it
// So check it more manually
got := fstest . NewDirectory ( ctx , t , r . Fremote , "sub dir" )
2024-03-01 10:56:48 +00:00
fstest . CheckDirModTime ( ctx , t , r . Fremote , got , subDirT )
2019-03-06 08:43:46 +00:00
}
2024-03-07 16:13:36 +00:00
// Test that --no-update-dir-modtime is working
func TestSyncNoUpdateDirModtime ( t * testing . T ) {
r := fstest . NewRun ( t )
if r . Fremote . Features ( ) . DirSetModTime == nil {
t . Skip ( "Skipping test as backend does not support DirSetModTime" )
}
ctx , ci := fs . AddConfig ( context . Background ( ) )
ci . NoUpdateDirModTime = true
const name = "sub dir no update dir modtime"
// Set the modtime on name to something specific
_ , err := operations . MkdirModTime ( ctx , r . Flocal , name , t1 )
require . NoError ( t , err )
// Create the remote directory with the current time
require . NoError ( t , r . Fremote . Mkdir ( ctx , name ) )
// Read its modification time
wantT := fstest . NewDirectory ( ctx , t , r . Fremote , name ) . ModTime ( ctx )
ctx = predictDstFromLogger ( ctx )
err = Sync ( ctx , r . Fremote , r . Flocal , true )
require . NoError ( t , err )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckRemoteListing (
t ,
[ ] fstest . Item { } ,
[ ] string {
name ,
} ,
)
// Read the new directory modification time - it should not have changed
gotT := fstest . NewDirectory ( ctx , t , r . Fremote , name ) . ModTime ( ctx )
fstest . AssertTimeEqualWithPrecision ( t , name , wantT , gotT , r . Fremote . Precision ( ) )
}
2024-04-04 17:03:20 +00:00
// Test move empty directories when we are not configured to create them
func TestMoveNoEmptyDirectories ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
err := operations . Mkdir ( ctx , r . Flocal , "sub dir2" )
require . NoError ( t , err )
r . Mkdir ( ctx , r . Fremote )
err = MoveDir ( ctx , r . Fremote , r . Flocal , false , false )
require . NoError ( t , err )
r . CheckRemoteListing (
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
} ,
)
}
2019-03-06 08:43:46 +00:00
// Test sync empty directories
func TestSyncEmptyDirectories ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2019-03-06 08:43:46 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
2024-02-06 16:00:34 +00:00
_ , err := operations . MkdirModTime ( ctx , r . Flocal , "sub dir2" , t2 )
2019-03-06 08:43:46 +00:00
require . NoError ( t , err )
2024-02-06 16:00:34 +00:00
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir" , t1 )
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2019-03-06 08:43:46 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , true )
2018-06-03 08:21:25 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-06-03 08:21:25 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2018-06-03 08:21:25 +00:00
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
"sub dir2" ,
} ,
)
2024-02-06 16:00:34 +00:00
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , "sub dir" , "sub dir2" )
}
// Test delayed mod time setting
func TestSyncSetDelayedModTimes ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
if ! r . Fremote . Features ( ) . DirModTimeUpdatesOnWrite {
t . Skip ( "Backend doesn't have DirModTimeUpdatesOnWrite set" )
}
// Create directories without timestamps
require . NoError ( t , r . Flocal . Mkdir ( ctx , "a1/b1/c1/d1/e1/f1" ) )
require . NoError ( t , r . Flocal . Mkdir ( ctx , "a1/b2/c1/d1/e1/f1" ) )
require . NoError ( t , r . Flocal . Mkdir ( ctx , "a1/b1/c1/d2/e1/f1" ) )
require . NoError ( t , r . Flocal . Mkdir ( ctx , "a1/b1/c1/d2/e1/f2" ) )
dirs := [ ] string {
"a1" ,
"a1/b1" ,
"a1/b1/c1" ,
"a1/b1/c1/d1" ,
"a1/b1/c1/d1/e1" ,
"a1/b1/c1/d1/e1/f1" ,
"a1/b1/c1/d2" ,
"a1/b1/c1/d2/e1" ,
"a1/b1/c1/d2/e1/f1" ,
"a1/b1/c1/d2/e1/f2" ,
"a1/b2" ,
"a1/b2/c1" ,
"a1/b2/c1/d1" ,
"a1/b2/c1/d1/e1" ,
"a1/b2/c1/d1/e1/f1" ,
}
r . CheckLocalListing ( t , [ ] fstest . Item { } , dirs )
// Timestamp the directories in reverse order
ts := t1
for i := len ( dirs ) - 1 ; i >= 0 ; i -- {
dir := dirs [ i ]
_ , err := operations . SetDirModTime ( ctx , r . Flocal , nil , dir , ts )
require . NoError ( t , err )
ts = ts . Add ( time . Minute )
}
r . Mkdir ( ctx , r . Fremote )
ctx = predictDstFromLogger ( ctx )
err := Sync ( ctx , r . Fremote , r . Flocal , true )
require . NoError ( t , err )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckRemoteListing ( t , [ ] fstest . Item { } , dirs )
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , dirs ... )
2018-06-03 08:21:25 +00:00
}
2024-04-04 17:03:20 +00:00
// Test sync empty directories when we are not configured to create them
func TestSyncNoEmptyDirectories ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
err := operations . Mkdir ( ctx , r . Flocal , "sub dir2" )
require . NoError ( t , err )
r . Mkdir ( ctx , r . Fremote )
err = Sync ( ctx , r . Fremote , r . Flocal , false )
require . NoError ( t , err )
r . CheckRemoteListing (
t ,
[ ] fstest . Item {
file1 ,
} ,
[ ] string {
"sub dir" ,
} ,
)
}
2020-10-13 21:43:40 +00:00
// Test a server-side copy if possible, or the backup path if not
2016-07-04 12:12:33 +00:00
func TestServerSideCopy ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "sub dir/hello world" , "hello world" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2019-08-08 18:58:02 +00:00
FremoteCopy , _ , finaliseCopy , err := fstest . RandomRemote ( )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
defer finaliseCopy ( )
2017-10-29 12:23:10 +00:00
t . Logf ( "Server side copy (if possible) %v -> %v" , r . Fremote , FremoteCopy )
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = CopyDir ( ctx , FremoteCopy , r . Fremote , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2017-10-29 12:23:10 +00:00
fstest . CheckItems ( t , FremoteCopy , file1 )
2016-07-04 12:12:33 +00:00
}
// Check that if the local file doesn't exist when we copy it up,
// nothing happens to the remote file
func TestCopyAfterDelete ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "sub dir/hello world" , "hello world" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2020-11-05 16:27:01 +00:00
err := operations . Mkdir ( ctx , r . Flocal , "" )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
// Check the copy downloading a file
func TestCopyRedownload ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "sub dir/hello world" , "hello world" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Flocal , r . Fremote , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2018-06-10 16:33:17 +00:00
// Test with combined precision of local and remote as we copied it there and back
2021-11-09 11:43:36 +00:00
r . CheckLocalListing ( t , [ ] fstest . Item { file1 } , nil )
2016-07-04 12:12:33 +00:00
}
// Create a file and sync it. Change the last modified date and resync.
// If we're only doing sync by size and checksum, we expect nothing to
// to be transferred on the second sync.
func TestSyncBasedOnCheckSum ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . CheckSum = true
2016-07-04 12:12:33 +00:00
2019-06-29 01:17:18 +00:00
file1 := r . WriteFile ( "check sum" , "-" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred exactly one file.
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
// Change last modified date only
2019-06-29 01:17:18 +00:00
file2 := r . WriteFile ( "check sum" , "-" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred no files
2019-07-18 10:13:54 +00:00
assert . Equal ( t , int64 ( 0 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
// Create a file and sync it. Change the last modified date and the
// file contents but not the size. If we're only doing sync by size
// only, we expect nothing to to be transferred on the second sync.
func TestSyncSizeOnly ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . SizeOnly = true
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "sizeonly" , "potato" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred exactly one file.
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
// Update mtime, md5sum but not length of file
file2 := r . WriteFile ( "sizeonly" , "POTATO" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred no files
2019-07-18 10:13:54 +00:00
assert . Equal ( t , int64 ( 0 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
// Create a file and sync it. Keep the last modified date but change
// the size. With --ignore-size we expect nothing to to be
// transferred on the second sync.
func TestSyncIgnoreSize ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . IgnoreSize = true
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "ignore-size" , "contents" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred exactly one file.
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
// Update size but not date of file
file2 := r . WriteFile ( "ignore-size" , "longer contents but same date" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred no files
2019-07-18 10:13:54 +00:00
assert . Equal ( t , int64 ( 0 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
func TestSyncIgnoreTimes ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteBoth ( ctx , "existing" , "potato" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred exactly 0 files because the
// files were identical.
2019-07-18 10:13:54 +00:00
assert . Equal ( t , int64 ( 0 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2016-07-04 12:12:33 +00:00
2020-11-05 11:33:32 +00:00
ci . IgnoreTimes = true
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// We should have transferred exactly one file even though the
// files were identical.
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
func TestSyncIgnoreExisting ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "existing" , "potato" , t1 )
2020-11-05 11:33:32 +00:00
ci . IgnoreExisting = true
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
// Change everything
r . WriteFile ( "existing" , "newpotatoes" , t2 )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
// Items should not change
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
}
2018-03-12 23:40:19 +00:00
func TestSyncIgnoreErrors ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2018-03-12 23:40:19 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . IgnoreErrors = true
2018-03-12 23:40:19 +00:00
file1 := r . WriteFile ( "a/potato2" , "------------------------------------------------------------" , t1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "b/potato" , "SMALLER BUT SAME DATE" , t2 )
file3 := r . WriteBoth ( ctx , "c/non empty space" , "AhHa!" , t2 )
require . NoError ( t , operations . Mkdir ( ctx , r . Fremote , "d" ) )
2018-03-12 23:40:19 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2018-03-12 23:40:19 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2018-03-12 23:40:19 +00:00
t ,
[ ] fstest . Item {
file2 ,
file3 ,
} ,
[ ] string {
"b" ,
"c" ,
"d" ,
} ,
)
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2019-11-18 14:13:02 +00:00
_ = fs . CountError ( errors . New ( "boom" ) )
2020-11-05 16:27:01 +00:00
assert . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-03-12 23:40:19 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2018-03-12 23:40:19 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2018-03-12 23:40:19 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
}
2016-07-04 12:12:33 +00:00
func TestSyncAfterChangingModtimeOnly ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2019-06-29 01:17:18 +00:00
file1 := r . WriteFile ( "empty space" , "-" , t2 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "empty space" , "-" , t1 )
2016-07-12 18:40:41 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2020-11-05 11:33:32 +00:00
ci . DryRun = true
2017-02-15 23:09:44 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2017-02-15 23:09:44 +00:00
2020-11-05 11:33:32 +00:00
ci . DryRun = false
2017-02-15 23:09:44 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2017-02-15 23:09:44 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-02-15 23:09:44 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2016-07-04 12:12:33 +00:00
}
2016-07-12 09:46:45 +00:00
func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-12-31 15:19:26 +00:00
2017-10-29 12:23:10 +00:00
if r . Fremote . Hashes ( ) . Count ( ) == 0 {
2017-02-09 11:01:20 +00:00
t . Logf ( "Can't check this if no hashes supported" )
2016-12-31 15:19:26 +00:00
return
}
2020-11-05 11:33:32 +00:00
ci . NoUpdateModTime = true
2016-07-12 09:46:45 +00:00
2019-06-29 01:17:18 +00:00
file1 := r . WriteFile ( "empty space" , "-" , t2 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "empty space" , "-" , t1 )
2016-07-12 09:46:45 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2016-07-12 18:40:41 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-12 09:46:45 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-12 09:46:45 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2016-07-12 09:46:45 +00:00
}
2016-11-28 17:08:15 +00:00
func TestSyncDoesntUpdateModtime ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
if fs . GetModifyWindow ( ctx , r . Fremote ) == fs . ModTimeNotSupported {
2018-06-03 18:45:34 +00:00
t . Skip ( "Can't run this test on fs which doesn't support mod time" )
}
2016-11-28 17:08:15 +00:00
file1 := r . WriteFile ( "foo" , "foo" , t2 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "foo" , "bar" , t1 )
2016-11-28 17:08:15 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2016-11-28 17:08:15 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-11-28 17:08:15 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-11-28 17:08:15 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2016-11-28 17:08:15 +00:00
// We should have transferred exactly one file, not set the mod time
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2016-11-28 17:08:15 +00:00
}
2016-07-04 12:12:33 +00:00
func TestSyncAfterAddingAFile ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
2016-07-04 12:12:33 +00:00
file2 := r . WriteFile ( "potato" , "------------------------------------------------------------" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file1 )
2016-07-12 18:40:41 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file1 , file2 )
2016-07-04 12:12:33 +00:00
}
func TestSyncAfterChangingFilesSizeOnly ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "potato" , "------------------------------------------------------------" , t3 )
2016-07-04 12:12:33 +00:00
file2 := r . WriteFile ( "potato" , "smaller but same date" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file2 )
2016-07-04 12:12:33 +00:00
}
// Sync after changing a file's contents, changing modtime but length
// remaining the same
func TestSyncAfterChangingContentsOnly ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
var file1 fstest . Item
2017-10-29 12:23:10 +00:00
if r . Fremote . Precision ( ) == fs . ModTimeNotSupported {
2016-07-04 12:12:33 +00:00
t . Logf ( "ModTimeNotSupported so forcing file to be a different size" )
2020-11-05 16:27:01 +00:00
file1 = r . WriteObject ( ctx , "potato" , "different size to make sure it syncs" , t3 )
2016-07-04 12:12:33 +00:00
} else {
2020-11-05 16:27:01 +00:00
file1 = r . WriteObject ( ctx , "potato" , "smaller but same date" , t3 )
2016-07-04 12:12:33 +00:00
}
file2 := r . WriteFile ( "potato" , "SMALLER BUT SAME DATE" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file2 )
2016-07-04 12:12:33 +00:00
}
// Sync after removing a file and adding a file --dry-run
func TestSyncAfterRemovingAFileAndAddingAFileDryRun ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "potato2" , "------------------------------------------------------------" , t1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "potato" , "SMALLER BUT SAME DATE" , t2 )
file3 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
2016-07-04 12:12:33 +00:00
2020-11-05 11:33:32 +00:00
ci . DryRun = true
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2020-11-05 11:33:32 +00:00
ci . DryRun = false
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file3 , file1 )
r . CheckRemoteItems ( t , file3 , file2 )
2016-07-04 12:12:33 +00:00
}
// Sync after removing a file and adding a file
2021-02-17 20:21:09 +00:00
func testSyncAfterRemovingAFileAndAddingAFile ( ctx context . Context , t * testing . T ) {
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-04 12:12:33 +00:00
file1 := r . WriteFile ( "potato2" , "------------------------------------------------------------" , t1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "potato" , "SMALLER BUT SAME DATE" , t2 )
file3 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 )
r . CheckLocalItems ( t , file1 , file3 )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2017-01-25 19:59:53 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file3 )
r . CheckRemoteItems ( t , file1 , file3 )
2017-01-25 19:59:53 +00:00
}
2021-02-17 20:21:09 +00:00
func TestSyncAfterRemovingAFileAndAddingAFile ( t * testing . T ) {
testSyncAfterRemovingAFileAndAddingAFile ( context . Background ( ) , t )
}
2017-01-25 19:59:53 +00:00
// Sync after removing a file and adding a file
2021-02-17 20:21:09 +00:00
func testSyncAfterRemovingAFileAndAddingAFileSubDir ( ctx context . Context , t * testing . T ) {
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-01-25 19:59:53 +00:00
file1 := r . WriteFile ( "a/potato2" , "------------------------------------------------------------" , t1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "b/potato" , "SMALLER BUT SAME DATE" , t2 )
file3 := r . WriteBoth ( ctx , "c/non empty space" , "AhHa!" , t2 )
require . NoError ( t , operations . Mkdir ( ctx , r . Fremote , "d" ) )
require . NoError ( t , operations . Mkdir ( ctx , r . Fremote , "d/e" ) )
2017-08-09 20:06:39 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file2 ,
file3 ,
} ,
[ ] string {
"b" ,
"c" ,
"d" ,
"d/e" ,
} ,
)
2017-01-25 19:59:53 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-08-09 20:06:39 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2016-07-04 12:12:33 +00:00
}
2021-02-17 20:21:09 +00:00
func TestSyncAfterRemovingAFileAndAddingAFileSubDir ( t * testing . T ) {
testSyncAfterRemovingAFileAndAddingAFileSubDir ( context . Background ( ) , t )
}
2016-07-04 12:12:33 +00:00
// Sync after removing a file and adding a file with IO Errors
2017-08-09 20:06:39 +00:00
func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-08-09 20:06:39 +00:00
file1 := r . WriteFile ( "a/potato2" , "------------------------------------------------------------" , t1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "b/potato" , "SMALLER BUT SAME DATE" , t2 )
file3 := r . WriteBoth ( ctx , "c/non empty space" , "AhHa!" , t2 )
require . NoError ( t , operations . Mkdir ( ctx , r . Fremote , "d" ) )
2017-08-09 20:06:39 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file2 ,
file3 ,
} ,
[ ] string {
"b" ,
"c" ,
"d" ,
} ,
)
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2019-11-18 14:13:02 +00:00
_ = fs . CountError ( errors . New ( "boom" ) )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
assert . Equal ( t , fs . ErrorNotDeleting , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-08-09 20:06:39 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file3 ,
} ,
[ ] string {
"a" ,
"c" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2017-08-09 20:06:39 +00:00
t ,
[ ] fstest . Item {
file1 ,
file2 ,
file3 ,
} ,
[ ] string {
"a" ,
"b" ,
"c" ,
"d" ,
} ,
)
2016-07-04 12:12:33 +00:00
}
2017-01-25 19:35:14 +00:00
// Sync test delete after
func TestSyncDeleteAfter ( t * testing . T ) {
2020-11-05 11:33:32 +00:00
ctx := context . Background ( )
ci := fs . GetConfig ( ctx )
2016-07-04 12:12:33 +00:00
// This is the default so we've checked this already
// check it is the default
2020-11-05 11:33:32 +00:00
require . Equal ( t , ci . DeleteMode , fs . DeleteModeAfter , "Didn't default to --delete-after" )
2016-07-04 12:12:33 +00:00
}
2017-01-25 19:35:14 +00:00
// Sync test delete during
func TestSyncDeleteDuring ( t * testing . T ) {
2020-11-05 11:33:32 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-11-05 11:33:32 +00:00
ci . DeleteMode = fs . DeleteModeDuring
2016-07-04 12:12:33 +00:00
2021-02-17 20:21:09 +00:00
testSyncAfterRemovingAFileAndAddingAFile ( ctx , t )
2016-07-04 12:12:33 +00:00
}
2017-01-25 19:35:14 +00:00
// Sync test delete before
func TestSyncDeleteBefore ( t * testing . T ) {
2020-11-05 11:33:32 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-11-05 11:33:32 +00:00
ci . DeleteMode = fs . DeleteModeBefore
2016-07-04 12:12:33 +00:00
2021-02-17 20:21:09 +00:00
testSyncAfterRemovingAFileAndAddingAFile ( ctx , t )
2016-07-04 12:12:33 +00:00
}
2017-02-22 13:17:38 +00:00
// Copy test delete before - shouldn't delete anything
func TestCopyDeleteBefore ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-02-22 13:17:38 +00:00
2020-11-05 11:33:32 +00:00
ci . DeleteMode = fs . DeleteModeBefore
2017-02-22 13:17:38 +00:00
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "potato" , "hopefully not deleted" , t1 )
2017-02-22 13:17:38 +00:00
file2 := r . WriteFile ( "potato2" , "hopefully copied in" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
r . CheckLocalItems ( t , file2 )
2017-02-22 13:17:38 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := CopyDir ( ctx , r . Fremote , r . Flocal , false )
2017-02-22 13:17:38 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-02-22 13:17:38 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 )
r . CheckLocalItems ( t , file2 )
2017-02-22 13:17:38 +00:00
}
2016-07-04 12:12:33 +00:00
// Test with exclude
func TestSyncWithExclude ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteBoth ( ctx , "potato2" , "------------------------------------------------------------" , t1 )
file2 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
2016-07-04 12:12:33 +00:00
file3 := r . WriteFile ( "enormous" , "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" , t1 ) // 100 bytes
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 )
r . CheckLocalItems ( t , file1 , file2 , file3 )
2016-07-04 12:12:33 +00:00
2021-02-17 20:21:09 +00:00
fi , err := filter . NewFilter ( nil )
require . NoError ( t , err )
2020-11-26 17:10:41 +00:00
fi . Opt . MaxSize = 40
2021-02-17 20:21:09 +00:00
ctx = filter . ReplaceConfig ( ctx , fi )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2021-02-17 20:21:09 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file1 )
2016-07-04 12:12:33 +00:00
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Flocal , r . Fremote , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 , file1 , file3 )
2016-07-04 12:12:33 +00:00
}
// Test with exclude and delete excluded
func TestSyncWithExcludeAndDeleteExcluded ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
file1 := r . WriteBoth ( ctx , "potato2" , "------------------------------------------------------------" , t1 ) // 60 bytes
file2 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
file3 := r . WriteBoth ( ctx , "enormous" , "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" , t1 ) // 100 bytes
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 )
r . CheckLocalItems ( t , file1 , file2 , file3 )
2016-07-04 12:12:33 +00:00
2021-02-17 20:21:09 +00:00
fi , err := filter . NewFilter ( nil )
require . NoError ( t , err )
2020-11-26 17:10:41 +00:00
fi . Opt . MaxSize = 40
fi . Opt . DeleteExcluded = true
2021-02-17 20:21:09 +00:00
ctx = filter . ReplaceConfig ( ctx , fi )
2016-07-04 12:12:33 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2021-02-17 20:21:09 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 )
2016-07-04 12:12:33 +00:00
// Check sync the other way round to make sure enormous gets
// deleted even though it is excluded
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Flocal , r . Fremote , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
2016-07-04 12:12:33 +00:00
}
// Test with UpdateOlder set
func TestSyncWithUpdateOlder ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2020-11-05 16:27:01 +00:00
if fs . GetModifyWindow ( ctx , r . Fremote ) == fs . ModTimeNotSupported {
2018-06-03 18:45:34 +00:00
t . Skip ( "Can't run this test on fs which doesn't support mod time" )
}
2016-07-04 12:12:33 +00:00
t2plus := t2 . Add ( time . Second / 2 )
t2minus := t2 . Add ( time . Second / 2 )
oneF := r . WriteFile ( "one" , "one" , t1 )
twoF := r . WriteFile ( "two" , "two" , t3 )
threeF := r . WriteFile ( "three" , "three" , t2 )
fourF := r . WriteFile ( "four" , "four" , t2 )
fiveF := r . WriteFile ( "five" , "five" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , oneF , twoF , threeF , fourF , fiveF )
2020-11-05 16:27:01 +00:00
oneO := r . WriteObject ( ctx , "one" , "ONE" , t2 )
twoO := r . WriteObject ( ctx , "two" , "TWO" , t2 )
threeO := r . WriteObject ( ctx , "three" , "THREE" , t2plus )
fourO := r . WriteObject ( ctx , "four" , "FOURFOUR" , t2minus )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , oneO , twoO , threeO , fourO )
2016-07-04 12:12:33 +00:00
2020-11-05 11:33:32 +00:00
ci . UpdateOlder = true
ci . ModifyWindow = fs . ModTimeNotSupported
2016-07-04 12:12:33 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , oneO , twoF , threeO , fourF , fiveF )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // no modtime
2019-06-08 13:08:23 +00:00
if r . Fremote . Hashes ( ) . Count ( ) == 0 {
t . Logf ( "Skip test with --checksum as no hashes supported" )
return
}
// now enable checksum
2020-11-05 11:33:32 +00:00
ci . CheckSum = true
2019-06-08 13:08:23 +00:00
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2019-06-08 13:08:23 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , oneO , twoF , threeF , fourF , fiveF )
2016-07-04 12:12:33 +00:00
}
2019-07-25 10:28:27 +00:00
// Test with a max transfer duration
2021-11-29 16:19:00 +00:00
func testSyncWithMaxDuration ( t * testing . T , cutoffMode fs . CutoffMode ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-01-26 09:20:03 +00:00
if * fstest . RemoteName != "" {
t . Skip ( "Skipping test on non local remote" )
}
2019-07-25 10:28:27 +00:00
r := fstest . NewRun ( t )
maxDuration := 250 * time . Millisecond
2020-11-05 11:33:32 +00:00
ci . MaxDuration = maxDuration
2021-11-29 16:19:00 +00:00
ci . CutoffMode = cutoffMode
ci . CheckFirst = true
ci . OrderBy = "size"
2020-11-05 11:33:32 +00:00
ci . Transfers = 1
2021-11-29 16:19:00 +00:00
ci . Checkers = 1
bytesPerSecond := 10 * 1024
accounting . TokenBucket . SetBwLimit ( fs . BwPair { Tx : fs . SizeSuffix ( bytesPerSecond ) , Rx : fs . SizeSuffix ( bytesPerSecond ) } )
2021-02-17 20:21:09 +00:00
defer accounting . TokenBucket . SetBwLimit ( fs . BwPair { Tx : - 1 , Rx : - 1 } )
2019-07-25 10:28:27 +00:00
2021-11-29 16:19:00 +00:00
// write one small file which we expect to transfer and one big one which we don't
file1 := r . WriteFile ( "file1" , string ( make ( [ ] byte , 16 ) ) , t1 )
file2 := r . WriteFile ( "file2" , string ( make ( [ ] byte , 50 * 1024 ) ) , t1 )
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t )
2019-07-25 10:28:27 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)
2019-07-25 10:28:27 +00:00
startTime := time . Now ( )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2023-07-15 16:41:13 +00:00
require . True ( t , errors . Is ( err , ErrorMaxDurationReached ) )
2019-07-25 10:28:27 +00:00
2021-11-29 16:19:00 +00:00
if cutoffMode == fs . CutoffModeHard {
r . CheckRemoteItems ( t , file1 )
assert . Equal ( t , int64 ( 1 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
} else {
r . CheckRemoteItems ( t , file1 , file2 )
assert . Equal ( t , int64 ( 2 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
}
2019-07-25 10:28:27 +00:00
elapsed := time . Since ( startTime )
2021-11-29 16:19:00 +00:00
const maxTransferTime = 20 * time . Second
2019-07-25 10:28:27 +00:00
what := fmt . Sprintf ( "expecting elapsed time %v between %v and %v" , elapsed , maxDuration , maxTransferTime )
2021-11-29 16:19:00 +00:00
assert . True ( t , elapsed >= maxDuration , what )
assert . True ( t , elapsed < maxTransferTime , what )
}
func TestSyncWithMaxDuration ( t * testing . T ) {
t . Run ( "Hard" , func ( t * testing . T ) {
testSyncWithMaxDuration ( t , fs . CutoffModeHard )
} )
t . Run ( "Soft" , func ( t * testing . T ) {
testSyncWithMaxDuration ( t , fs . CutoffModeSoft )
} )
2019-07-25 10:28:27 +00:00
}
2016-12-18 10:03:56 +00:00
// Test with TrackRenames set
func TestSyncWithTrackRenames ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-12-18 10:03:56 +00:00
2020-11-05 11:33:32 +00:00
ci . TrackRenames = true
2016-12-18 10:03:56 +00:00
defer func ( ) {
2020-11-05 11:33:32 +00:00
ci . TrackRenames = false
2016-12-18 10:03:56 +00:00
} ( )
2018-01-18 20:27:52 +00:00
haveHash := r . Fremote . Hashes ( ) . Overlap ( r . Flocal . Hashes ( ) ) . GetOne ( ) != hash . None
2018-01-12 16:30:54 +00:00
canTrackRenames := haveHash && operations . CanServerSideMove ( r . Fremote )
2017-01-03 17:35:12 +00:00
t . Logf ( "Can track renames: %v" , canTrackRenames )
2016-12-18 10:03:56 +00:00
f1 := r . WriteFile ( "potato" , "Potato Content" , t1 )
f2 := r . WriteFile ( "yam" , "Yam Content" , t2 )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-12-18 10:03:56 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
r . CheckLocalItems ( t , f1 , f2 )
2016-12-18 10:03:56 +00:00
// Now rename locally.
f2 = r . RenameFile ( f2 , "yaml" )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-12-18 10:03:56 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
2016-12-18 10:03:56 +00:00
2020-03-31 09:58:49 +00:00
// Check we renamed something if we should have
if canTrackRenames {
renames := accounting . GlobalStats ( ) . Renames ( 0 )
assert . Equal ( t , canTrackRenames , renames != 0 , fmt . Sprintf ( "canTrackRenames=%v, renames=%d" , canTrackRenames , renames ) )
}
2020-03-20 13:04:56 +00:00
}
2020-03-21 17:35:34 +00:00
func TestParseRenamesStrategyModtime ( t * testing . T ) {
for _ , test := range [ ] struct {
in string
want trackRenamesStrategy
wantErr bool
} {
{ "" , 0 , false } ,
{ "modtime" , trackRenamesStrategyModtime , false } ,
{ "hash" , trackRenamesStrategyHash , false } ,
{ "size" , 0 , false } ,
{ "modtime,hash" , trackRenamesStrategyModtime | trackRenamesStrategyHash , false } ,
{ "hash,modtime,size" , trackRenamesStrategyModtime | trackRenamesStrategyHash , false } ,
{ "size,boom" , 0 , true } ,
} {
got , err := parseTrackRenamesStrategy ( test . in )
assert . Equal ( t , test . want , got , test . in )
assert . Equal ( t , test . wantErr , err != nil , test . in )
}
}
func TestRenamesStrategyModtime ( t * testing . T ) {
both := trackRenamesStrategyHash | trackRenamesStrategyModtime
hash := trackRenamesStrategyHash
modTime := trackRenamesStrategyModtime
assert . True ( t , both . hash ( ) )
assert . True ( t , both . modTime ( ) )
assert . True ( t , hash . hash ( ) )
assert . False ( t , hash . modTime ( ) )
assert . False ( t , modTime . hash ( ) )
assert . True ( t , modTime . modTime ( ) )
}
2020-03-20 13:04:56 +00:00
func TestSyncWithTrackRenamesStrategyModtime ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-03-20 13:04:56 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . TrackRenames = true
ci . TrackRenamesStrategy = "modtime"
2020-03-20 13:04:56 +00:00
2020-03-22 11:52:40 +00:00
canTrackRenames := operations . CanServerSideMove ( r . Fremote ) && r . Fremote . Precision ( ) != fs . ModTimeNotSupported
2020-03-20 13:04:56 +00:00
t . Logf ( "Can track renames: %v" , canTrackRenames )
f1 := r . WriteFile ( "potato" , "Potato Content" , t1 )
f2 := r . WriteFile ( "yam" , "Yam Content" , t2 )
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2020-03-20 13:04:56 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
r . CheckLocalItems ( t , f1 , f2 )
2020-03-20 13:04:56 +00:00
// Now rename locally.
f2 = r . RenameFile ( f2 , "yaml" )
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2020-03-20 13:04:56 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
2020-06-09 19:40:03 +00:00
// Check we renamed something if we should have
if canTrackRenames {
renames := accounting . GlobalStats ( ) . Renames ( 0 )
assert . Equal ( t , canTrackRenames , renames != 0 , fmt . Sprintf ( "canTrackRenames=%v, renames=%d" , canTrackRenames , renames ) )
}
}
func TestSyncWithTrackRenamesStrategyLeaf ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-06-09 19:40:03 +00:00
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . TrackRenames = true
ci . TrackRenamesStrategy = "leaf"
2020-06-09 19:40:03 +00:00
canTrackRenames := operations . CanServerSideMove ( r . Fremote ) && r . Fremote . Precision ( ) != fs . ModTimeNotSupported
t . Logf ( "Can track renames: %v" , canTrackRenames )
f1 := r . WriteFile ( "potato" , "Potato Content" , t1 )
f2 := r . WriteFile ( "sub/yam" , "Yam Content" , t2 )
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2020-06-09 19:40:03 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
r . CheckLocalItems ( t , f1 , f2 )
2020-06-09 19:40:03 +00:00
// Now rename locally.
f2 = r . RenameFile ( f2 , "yam" )
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
require . NoError ( t , Sync ( ctx , r . Fremote , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2020-06-09 19:40:03 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , f1 , f2 )
2020-03-20 13:04:56 +00:00
2020-03-31 09:58:49 +00:00
// Check we renamed something if we should have
if canTrackRenames {
renames := accounting . GlobalStats ( ) . Renames ( 0 )
assert . Equal ( t , canTrackRenames , renames != 0 , fmt . Sprintf ( "canTrackRenames=%v, renames=%d" , canTrackRenames , renames ) )
}
2019-09-05 20:29:35 +00:00
}
func toyFileTransfers ( r * fstest . Run ) int64 {
remote := r . Fremote . Name ( )
transfers := 1
if strings . HasPrefix ( remote , "TestChunker" ) && strings . HasSuffix ( remote , "S3" ) {
transfers ++ // Extra Copy because S3 emulates Move as Copy+Delete.
2017-01-03 17:35:12 +00:00
}
2019-09-05 20:29:35 +00:00
return int64 ( transfers )
2016-12-18 10:03:56 +00:00
}
2020-10-13 21:43:40 +00:00
// Test a server-side move if possible, or the backup path if not
2021-02-17 20:21:09 +00:00
func testServerSideMove ( ctx context . Context , t * testing . T , r * fstest . Run , withFilter , testDeleteEmptyDirs bool ) {
2019-08-08 18:58:02 +00:00
FremoteMove , _ , finaliseMove , err := fstest . RandomRemote ( )
2017-08-04 16:08:16 +00:00
require . NoError ( t , err )
defer finaliseMove ( )
2020-11-05 16:27:01 +00:00
file1 := r . WriteBoth ( ctx , "potato2" , "------------------------------------------------------------" , t1 )
file2 := r . WriteBoth ( ctx , "empty space" , "-" , t2 )
file3u := r . WriteBoth ( ctx , "potato3" , "------------------------------------------------------------ UPDATED" , t2 )
2016-07-04 12:12:33 +00:00
2017-11-27 11:42:02 +00:00
if testDeleteEmptyDirs {
2020-11-05 16:27:01 +00:00
err := operations . Mkdir ( ctx , r . Fremote , "tomatoDir" )
2017-11-27 11:42:02 +00:00
require . NoError ( t , err )
}
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file1 , file3u )
2016-07-04 12:12:33 +00:00
2017-10-29 12:23:10 +00:00
t . Logf ( "Server side move (if possible) %v -> %v" , r . Fremote , FremoteMove )
2016-07-04 12:12:33 +00:00
// Write just one file in the new remote
2020-11-05 16:27:01 +00:00
r . WriteObjectTo ( ctx , FremoteMove , "empty space" , "-" , t2 , false )
file3 := r . WriteObjectTo ( ctx , FremoteMove , "potato3" , "------------------------------------------------------------" , t1 , false )
2017-10-29 12:23:10 +00:00
fstest . CheckItems ( t , FremoteMove , file2 , file3 )
2016-07-04 12:12:33 +00:00
2020-10-13 21:43:40 +00:00
// Do server-side move
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx) // not currently supported -- doesn't list all contents of dir.
2020-11-05 16:27:01 +00:00
err = MoveDir ( ctx , FremoteMove , r . Fremote , testDeleteEmptyDirs , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2016-07-04 12:12:33 +00:00
2016-10-03 18:58:44 +00:00
if withFilter {
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 )
2016-10-03 18:58:44 +00:00
} else {
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t )
2016-10-03 18:58:44 +00:00
}
2017-11-27 11:42:02 +00:00
if testDeleteEmptyDirs {
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing ( t , nil , [ ] string { } )
2017-11-27 11:42:02 +00:00
}
2017-10-29 12:23:10 +00:00
fstest . CheckItems ( t , FremoteMove , file2 , file1 , file3u )
2016-07-04 12:12:33 +00:00
2017-08-04 16:08:16 +00:00
// Create a new empty remote for stuff to be moved into
2019-08-08 18:58:02 +00:00
FremoteMove2 , _ , finaliseMove2 , err := fstest . RandomRemote ( )
2017-08-04 16:08:16 +00:00
require . NoError ( t , err )
defer finaliseMove2 ( )
2016-07-11 10:36:46 +00:00
2017-11-27 11:42:02 +00:00
if testDeleteEmptyDirs {
2020-11-05 16:27:01 +00:00
err := operations . Mkdir ( ctx , FremoteMove , "tomatoDir" )
2017-11-27 11:42:02 +00:00
require . NoError ( t , err )
}
2017-08-04 16:08:16 +00:00
// Move it back to a new empty remote, dst does not exist this time
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = MoveDir ( ctx , FremoteMove2 , FremoteMove , testDeleteEmptyDirs , false )
2016-07-04 12:12:33 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2016-07-04 12:12:33 +00:00
2016-07-11 10:36:46 +00:00
if withFilter {
2017-10-29 12:23:10 +00:00
fstest . CheckItems ( t , FremoteMove2 , file1 , file3u )
fstest . CheckItems ( t , FremoteMove , file2 )
2016-07-11 10:36:46 +00:00
} else {
2017-10-29 12:23:10 +00:00
fstest . CheckItems ( t , FremoteMove2 , file2 , file1 , file3u )
2017-11-23 05:22:01 +00:00
fstest . CheckItems ( t , FremoteMove )
2016-07-11 10:36:46 +00:00
}
2017-11-27 11:42:02 +00:00
if testDeleteEmptyDirs {
2020-11-05 16:27:01 +00:00
fstest . CheckListingWithPrecision ( t , FremoteMove , nil , [ ] string { } , fs . GetModifyWindow ( ctx , r . Fremote ) )
2017-11-27 11:42:02 +00:00
}
2016-07-11 10:36:46 +00:00
}
2023-11-15 20:12:45 +00:00
// Test MoveDir on Local
func TestServerSideMoveLocal ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
f1 := r . WriteFile ( "dir1/file1.txt" , "hello" , t1 )
f2 := r . WriteFile ( "dir2/file2.txt" , "hello again" , t2 )
r . CheckLocalItems ( t , f1 , f2 )
dir1 , err := fs . NewFs ( ctx , r . Flocal . Root ( ) + "/dir1" )
require . NoError ( t , err )
dir2 , err := fs . NewFs ( ctx , r . Flocal . Root ( ) + "/dir2" )
require . NoError ( t , err )
err = MoveDir ( ctx , dir2 , dir1 , false , false )
require . NoError ( t , err )
}
2018-07-17 05:43:58 +00:00
// Test move
func TestMoveWithDeleteEmptySrcDirs ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2018-07-17 05:43:58 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
file2 := r . WriteFile ( "nested/sub dir/file" , "nested" , t1 )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2018-07-17 05:43:58 +00:00
// run move with --delete-empty-src-dirs
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := MoveDir ( ctx , r . Fremote , r . Flocal , true , false )
2018-07-17 05:43:58 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-07-17 05:43:58 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2018-07-17 05:43:58 +00:00
t ,
nil ,
[ ] string { } ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 )
2018-07-17 05:43:58 +00:00
}
func TestMoveWithoutDeleteEmptySrcDirs ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2018-07-17 05:43:58 +00:00
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
file2 := r . WriteFile ( "nested/sub dir/file" , "nested" , t1 )
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2018-07-17 05:43:58 +00:00
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := MoveDir ( ctx , r . Fremote , r . Flocal , false , false )
2018-07-17 05:43:58 +00:00
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2018-07-17 05:43:58 +00:00
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2018-07-17 05:43:58 +00:00
t ,
nil ,
[ ] string {
"sub dir" ,
"nested" ,
"nested/sub dir" ,
} ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 )
2018-07-17 05:43:58 +00:00
}
2021-07-29 16:42:55 +00:00
func TestMoveWithIgnoreExisting ( t * testing . T ) {
ctx := context . Background ( )
ctx , ci := fs . AddConfig ( ctx )
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "existing" , "potato" , t1 )
file2 := r . WriteFile ( "existing-b" , "tomato" , t1 )
ci . IgnoreExisting = true
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2021-07-29 16:42:55 +00:00
err := MoveDir ( ctx , r . Fremote , r . Flocal , false , false )
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2021-07-29 16:42:55 +00:00
t ,
[ ] fstest . Item { } ,
[ ] string { } ,
)
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2021-07-29 16:42:55 +00:00
t ,
[ ] fstest . Item {
file1 ,
file2 ,
} ,
[ ] string { } ,
)
// Recreate first file with modified content
file1b := r . WriteFile ( "existing" , "newpotatoes" , t2 )
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2021-07-29 16:42:55 +00:00
err = MoveDir ( ctx , r . Fremote , r . Flocal , false , false )
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-07-29 16:42:55 +00:00
// Source items should still exist in modified state
2021-11-09 11:43:36 +00:00
r . CheckLocalListing (
2021-07-29 16:42:55 +00:00
t ,
[ ] fstest . Item {
file1b ,
} ,
[ ] string { } ,
)
// Dest items should not have changed
2021-11-09 11:43:36 +00:00
r . CheckRemoteListing (
2021-07-29 16:42:55 +00:00
t ,
[ ] fstest . Item {
file1 ,
file2 ,
} ,
[ ] string { } ,
)
}
2020-10-13 21:43:40 +00:00
// Test a server-side move if possible, or the backup path if not
2016-07-11 10:36:46 +00:00
func TestServerSideMove ( t * testing . T ) {
2021-02-17 20:21:09 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2021-02-17 20:21:09 +00:00
testServerSideMove ( ctx , t , r , false , false )
2016-07-11 10:36:46 +00:00
}
2020-10-13 21:43:40 +00:00
// Test a server-side move if possible, or the backup path if not
2016-07-11 10:36:46 +00:00
func TestServerSideMoveWithFilter ( t * testing . T ) {
2020-11-26 17:10:41 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-07-11 10:36:46 +00:00
2021-02-17 20:21:09 +00:00
fi , err := filter . NewFilter ( nil )
require . NoError ( t , err )
2020-11-26 17:10:41 +00:00
fi . Opt . MinSize = 40
2021-02-17 20:21:09 +00:00
ctx = filter . ReplaceConfig ( ctx , fi )
2016-07-11 10:36:46 +00:00
2021-02-17 20:21:09 +00:00
testServerSideMove ( ctx , t , r , true , false )
2017-11-27 11:42:02 +00:00
}
2020-10-13 21:43:40 +00:00
// Test a server-side move if possible
2017-11-27 11:42:02 +00:00
func TestServerSideMoveDeleteEmptySourceDirs ( t * testing . T ) {
2021-02-17 20:21:09 +00:00
ctx := context . Background ( )
2017-11-27 11:42:02 +00:00
r := fstest . NewRun ( t )
2021-02-17 20:21:09 +00:00
testServerSideMove ( ctx , t , r , false , true )
2016-07-11 10:36:46 +00:00
}
2020-10-13 21:43:40 +00:00
// Test a server-side move with overlap
2016-07-11 10:36:46 +00:00
func TestServerSideMoveOverlap ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2016-10-22 16:53:52 +00:00
2017-10-29 12:23:10 +00:00
if r . Fremote . Features ( ) . DirMove != nil {
2016-10-22 16:53:52 +00:00
t . Skip ( "Skipping test as remote supports DirMove" )
}
2017-10-29 12:23:10 +00:00
subRemoteName := r . FremoteName + "/rclone-move-test"
2020-11-05 16:27:01 +00:00
FremoteMove , err := fs . NewFs ( ctx , subRemoteName )
2016-07-11 10:36:46 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "potato2" , "------------------------------------------------------------" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2016-07-11 10:36:46 +00:00
// Subdir move with no filters should return ErrorCantMoveOverlapping
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = MoveDir ( ctx , FremoteMove , r . Fremote , false , false )
2019-02-14 12:06:26 +00:00
assert . EqualError ( t , err , fs . ErrorOverlapping . Error ( ) )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2016-07-11 10:36:46 +00:00
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
2021-02-17 20:21:09 +00:00
fi , err := filter . NewFilter ( nil )
require . NoError ( t , err )
2020-11-26 17:10:41 +00:00
fi . Opt . MinSize = 40
2021-02-17 20:21:09 +00:00
ctx = filter . ReplaceConfig ( ctx , fi )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = MoveDir ( ctx , FremoteMove , r . Fremote , false , false )
2019-02-27 17:13:13 +00:00
assert . EqualError ( t , err , fs . ErrorOverlapping . Error ( ) )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-02-14 12:06:26 +00:00
}
// Test a sync with overlap
func TestSyncOverlap ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2019-02-14 12:06:26 +00:00
r := fstest . NewRun ( t )
subRemoteName := r . FremoteName + "/rclone-sync-test"
2020-11-05 16:27:01 +00:00
FremoteSync , err := fs . NewFs ( ctx , subRemoteName )
2019-02-14 12:06:26 +00:00
require . NoError ( t , err )
checkErr := func ( err error ) {
2019-02-28 11:39:32 +00:00
require . Error ( t , err )
2019-02-14 12:06:26 +00:00
assert . True ( t , fserrors . IsFatalError ( err ) )
2019-02-28 11:39:32 +00:00
assert . Equal ( t , fs . ErrorOverlapping . Error ( ) , err . Error ( ) )
2019-02-14 12:06:26 +00:00
}
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
checkErr ( Sync ( ctx , FremoteSync , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
checkErr ( Sync ( ctx , r . Fremote , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
checkErr ( Sync ( ctx , r . Fremote , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
checkErr ( Sync ( ctx , FremoteSync , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2016-07-04 12:12:33 +00:00
}
2017-01-10 21:47:03 +00:00
2022-06-01 17:24:54 +00:00
// Test a sync with filtered overlap
func TestSyncOverlapWithFilter ( t * testing . T ) {
ctx := context . Background ( )
r := fstest . NewRun ( t )
fi , err := filter . NewFilter ( nil )
require . NoError ( t , err )
require . NoError ( t , fi . Add ( false , "/rclone-sync-test/" ) )
require . NoError ( t , fi . Add ( false , "*/layer2/" ) )
2022-06-08 07:29:01 +00:00
fi . Opt . ExcludeFile = [ ] string { ".ignore" }
2023-07-15 08:10:26 +00:00
filterCtx := filter . ReplaceConfig ( ctx , fi )
2022-06-01 17:24:54 +00:00
subRemoteName := r . FremoteName + "/rclone-sync-test"
FremoteSync , err := fs . NewFs ( ctx , subRemoteName )
require . NoError ( t , FremoteSync . Mkdir ( ctx , "" ) )
require . NoError ( t , err )
subRemoteName2 := r . FremoteName + "/rclone-sync-test-include/layer2"
FremoteSync2 , err := fs . NewFs ( ctx , subRemoteName2 )
require . NoError ( t , FremoteSync2 . Mkdir ( ctx , "" ) )
require . NoError ( t , err )
subRemoteName3 := r . FremoteName + "/rclone-sync-test-ignore-file"
FremoteSync3 , err := fs . NewFs ( ctx , subRemoteName3 )
require . NoError ( t , FremoteSync3 . Mkdir ( ctx , "" ) )
require . NoError ( t , err )
2022-06-09 12:19:11 +00:00
r . WriteObject ( context . Background ( ) , "rclone-sync-test-ignore-file/.ignore" , "-" , t1 )
2022-06-01 17:24:54 +00:00
checkErr := func ( err error ) {
require . Error ( t , err )
assert . True ( t , fserrors . IsFatalError ( err ) )
assert . Equal ( t , fs . ErrorOverlapping . Error ( ) , err . Error ( ) )
2022-06-09 12:19:11 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2022-06-01 17:24:54 +00:00
}
checkNoErr := func ( err error ) {
require . NoError ( t , err )
}
2022-06-09 12:19:11 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkNoErr ( Sync ( filterCtx , FremoteSync , r . Fremote , false ) )
checkErr ( Sync ( ctx , FremoteSync , r . Fremote , false ) )
checkNoErr ( Sync ( filterCtx , r . Fremote , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , r . Fremote , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( filterCtx , r . Fremote , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , r . Fremote , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( filterCtx , FremoteSync , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , FremoteSync , FremoteSync , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
2023-07-15 08:10:26 +00:00
checkNoErr ( Sync ( filterCtx , FremoteSync2 , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( ctx , FremoteSync2 , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkNoErr ( Sync ( filterCtx , r . Fremote , FremoteSync2 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , r . Fremote , FremoteSync2 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( filterCtx , FremoteSync2 , FremoteSync2 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , FremoteSync2 , FremoteSync2 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
2023-07-15 08:10:26 +00:00
checkNoErr ( Sync ( filterCtx , FremoteSync3 , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( ctx , FremoteSync3 , r . Fremote , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-08-13 10:39:02 +00:00
// Destination is excluded so this test makes no sense
// checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false))
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , r . Fremote , FremoteSync3 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2023-07-15 08:10:26 +00:00
checkErr ( Sync ( filterCtx , FremoteSync3 , FremoteSync3 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
filterCtx = predictDstFromLogger ( filterCtx )
2022-06-01 17:24:54 +00:00
checkErr ( Sync ( ctx , FremoteSync3 , FremoteSync3 , false ) )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( filterCtx , r . Fremote , operations . GetLoggerOpt ( filterCtx ) . JSON , t )
2022-06-01 17:24:54 +00:00
}
2019-07-08 01:02:53 +00:00
// Test with CompareDest set
func TestSyncCompareDest ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2019-07-08 01:02:53 +00:00
r := fstest . NewRun ( t )
2020-11-16 03:04:29 +00:00
ci . CompareDest = [ ] string { r . FremoteName + "/CompareDest" }
2019-07-08 01:02:53 +00:00
2020-11-05 16:27:01 +00:00
fdst , err := fs . NewFs ( ctx , r . FremoteName + "/dst" )
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
// check empty dest, empty compare
file1 := r . WriteFile ( "one" , "one" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx) // not currently supported due to duplicate equal() checks
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file1dst := file1
file1dst . Path = "dst/one"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1dst )
2019-07-08 01:02:53 +00:00
// check old dest, empty compare
file1b := r . WriteFile ( "one" , "onet2" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1dst )
r . CheckLocalItems ( t , file1b )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file1bdst := file1b
file1bdst . Path = "dst/one"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1bdst )
2019-07-08 01:02:53 +00:00
// check old dest, new compare
2020-11-05 16:27:01 +00:00
file3 := r . WriteObject ( ctx , "dst/one" , "one" , t1 )
file2 := r . WriteObject ( ctx , "CompareDest/one" , "onet2" , t2 )
2019-07-08 01:02:53 +00:00
file1c := r . WriteFile ( "one" , "onet2" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 )
r . CheckLocalItems ( t , file1c )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 )
2019-07-08 01:02:53 +00:00
// check empty dest, new compare
2020-11-05 16:27:01 +00:00
file4 := r . WriteObject ( ctx , "CompareDest/two" , "two" , t2 )
2019-07-08 01:02:53 +00:00
file5 := r . WriteFile ( "two" , "two" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 )
r . CheckLocalItems ( t , file1c , file5 )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 )
2019-07-08 01:02:53 +00:00
// check new dest, new compare
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 )
2019-07-08 01:02:53 +00:00
2021-07-07 14:08:47 +00:00
// Work out if we actually have hashes for uploaded files
haveHash := false
if ht := fdst . Hashes ( ) . GetOne ( ) ; ht != hash . None {
file2obj , err := fdst . NewObject ( ctx , "one" )
if err == nil {
file2objHash , err := file2obj . Hash ( ctx , ht )
if err == nil {
haveHash = file2objHash != ""
}
}
}
2021-06-14 20:19:12 +00:00
2021-07-07 14:08:47 +00:00
// check new dest, new compare, src timestamp differs
//
// we only check this if we the file we uploaded previously
// actually has a hash otherwise the differing timestamp is
// always copied.
if haveHash {
file5b := r . WriteFile ( "two" , "two" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1c , file5b )
2021-07-07 14:08:47 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2021-07-07 14:08:47 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2021-07-07 14:08:47 +00:00
require . NoError ( t , err )
2021-06-14 20:19:12 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 )
2021-07-07 14:08:47 +00:00
} else {
t . Log ( "No hash on uploaded file so skipping compare timestamp test" )
}
2021-06-14 20:19:12 +00:00
2019-07-08 01:02:53 +00:00
// check empty dest, old compare
2021-06-14 20:19:12 +00:00
file5c := r . WriteFile ( "two" , "twot3" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 )
r . CheckLocalItems ( t , file1c , file5c )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
2021-06-14 20:19:12 +00:00
file5cdst := file5c
file5cdst . Path = "dst/two"
2019-07-08 01:02:53 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 , file4 , file5cdst )
2019-07-08 01:02:53 +00:00
}
2020-11-16 03:04:29 +00:00
// Test with multiple CompareDest
func TestSyncMultipleCompareDest ( t * testing . T ) {
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-11-16 03:04:29 +00:00
r := fstest . NewRun ( t )
2021-11-09 11:43:36 +00:00
precision := fs . GetModifyWindow ( ctx , r . Fremote , r . Flocal )
2020-11-16 03:04:29 +00:00
ci . CompareDest = [ ] string { r . FremoteName + "/pre-dest1" , r . FremoteName + "/pre-dest2" }
// check empty dest, new compare
fsrc1 := r . WriteFile ( "1" , "1" , t1 )
fsrc2 := r . WriteFile ( "2" , "2" , t1 )
fsrc3 := r . WriteFile ( "3" , "3" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , fsrc1 , fsrc2 , fsrc3 )
2020-11-16 03:04:29 +00:00
fdest1 := r . WriteObject ( ctx , "pre-dest1/1" , "1" , t1 )
fdest2 := r . WriteObject ( ctx , "pre-dest2/2" , "2" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , fdest1 , fdest2 )
2020-11-16 03:04:29 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
fdst , err := fs . NewFs ( ctx , r . FremoteName + "/dest" )
require . NoError ( t , err )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-16 03:04:29 +00:00
require . NoError ( t , Sync ( ctx , fdst , r . Flocal , false ) )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
2020-11-16 03:04:29 +00:00
fdest3 := fsrc3
fdest3 . Path = "dest/3"
2021-11-09 11:43:36 +00:00
fstest . CheckItemsWithPrecision ( t , fdst , precision , fsrc3 )
r . CheckRemoteItems ( t , fdest1 , fdest2 , fdest3 )
2020-11-16 03:04:29 +00:00
}
2019-07-08 01:02:53 +00:00
// Test with CopyDest set
func TestSyncCopyDest ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2019-07-08 01:02:53 +00:00
r := fstest . NewRun ( t )
if r . Fremote . Features ( ) . Copy == nil {
2020-10-13 21:43:40 +00:00
t . Skip ( "Skipping test as remote does not support server-side copy" )
2019-07-08 01:02:53 +00:00
}
2020-11-16 03:04:29 +00:00
ci . CopyDest = [ ] string { r . FremoteName + "/CopyDest" }
2019-07-08 01:02:53 +00:00
2020-11-05 16:27:01 +00:00
fdst , err := fs . NewFs ( ctx , r . FremoteName + "/dst" )
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
// check empty dest, empty copy
file1 := r . WriteFile ( "one" , "one" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file1dst := file1
file1dst . Path = "dst/one"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1dst )
2019-07-08 01:02:53 +00:00
// check old dest, empty copy
file1b := r . WriteFile ( "one" , "onet2" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1dst )
r . CheckLocalItems ( t , file1b )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file1bdst := file1b
file1bdst . Path = "dst/one"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1bdst )
2019-07-08 01:02:53 +00:00
// check old dest, new copy, backup-dir
2020-11-05 11:33:32 +00:00
ci . BackupDir = r . FremoteName + "/BackupDir"
2019-07-08 01:02:53 +00:00
2020-11-05 16:27:01 +00:00
file3 := r . WriteObject ( ctx , "dst/one" , "one" , t1 )
file2 := r . WriteObject ( ctx , "CopyDest/one" , "onet2" , t2 )
2019-07-08 01:02:53 +00:00
file1c := r . WriteFile ( "one" , "onet2" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file3 )
r . CheckLocalItems ( t , file1c )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file2dst := file2
file2dst . Path = "dst/one"
file3 . Path = "BackupDir/one"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 )
2020-11-05 11:33:32 +00:00
ci . BackupDir = ""
2019-07-08 01:02:53 +00:00
// check empty dest, new copy
2020-11-05 16:27:01 +00:00
file4 := r . WriteObject ( ctx , "CopyDest/two" , "two" , t2 )
2019-07-08 01:02:53 +00:00
file5 := r . WriteFile ( "two" , "two" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 , file4 )
r . CheckLocalItems ( t , file1c , file5 )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file4dst := file4
file4dst . Path = "dst/two"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 , file4 , file4dst )
2019-07-08 01:02:53 +00:00
// check new dest, new copy
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 , file4 , file4dst )
2019-07-08 01:02:53 +00:00
// check empty dest, old copy
2020-11-05 16:27:01 +00:00
file6 := r . WriteObject ( ctx , "CopyDest/three" , "three" , t2 )
2019-07-08 01:02:53 +00:00
file7 := r . WriteFile ( "three" , "threet3" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 , file4 , file4dst , file6 )
r . CheckLocalItems ( t , file1c , file5 , file7 )
2019-07-08 01:02:53 +00:00
2019-07-22 19:11:46 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-11-06 07:45:51 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2023-11-06 07:45:51 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2019-07-08 01:02:53 +00:00
require . NoError ( t , err )
file7dst := file7
file7dst . Path = "dst/three"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 , file2dst , file3 , file4 , file4dst , file6 , file7dst )
2019-07-08 01:02:53 +00:00
}
2017-01-10 21:47:03 +00:00
// Test with BackupDir set
2020-10-04 15:38:29 +00:00
func testSyncBackupDir ( t * testing . T , backupDir string , suffix string , suffixKeepExtension bool ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-01-10 21:47:03 +00:00
2018-01-12 16:30:54 +00:00
if ! operations . CanServerSideMove ( r . Fremote ) {
2020-10-13 21:43:40 +00:00
t . Skip ( "Skipping test as remote does not support server-side move" )
2017-01-10 21:47:03 +00:00
}
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2017-01-10 21:47:03 +00:00
2020-10-04 15:38:29 +00:00
if backupDir != "" {
2020-11-05 11:33:32 +00:00
ci . BackupDir = r . FremoteName + "/" + backupDir
2020-10-04 15:38:29 +00:00
backupDir += "/"
} else {
2020-11-05 11:33:32 +00:00
ci . BackupDir = ""
2020-10-04 15:38:29 +00:00
backupDir = "dst/"
// Exclude the suffix from the sync otherwise the sync
// deletes the old backup files
flt , err := filter . NewFilter ( nil )
require . NoError ( t , err )
require . NoError ( t , flt . AddRule ( "- *" + suffix ) )
2020-11-26 17:10:41 +00:00
// Change the active filter
ctx = filter . ReplaceConfig ( ctx , flt )
2020-10-04 15:38:29 +00:00
}
2020-11-05 11:33:32 +00:00
ci . Suffix = suffix
ci . SuffixKeepExtension = suffixKeepExtension
2017-01-10 21:47:03 +00:00
2017-01-19 17:26:29 +00:00
// Make the setup so we have one, two, three in the dest
// and one (different), two (same) in the source
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "dst/one" , "one" , t1 )
file2 := r . WriteObject ( ctx , "dst/two" , "two" , t1 )
file3 := r . WriteObject ( ctx , "dst/three.txt" , "three" , t1 )
2017-01-19 17:26:29 +00:00
file2a := r . WriteFile ( "two" , "two" , t1 )
file1a := r . WriteFile ( "one" , "oneA" , t2 )
2017-01-10 21:47:03 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 )
r . CheckLocalItems ( t , file1a , file2a )
2017-01-10 21:47:03 +00:00
2020-11-05 16:27:01 +00:00
fdst , err := fs . NewFs ( ctx , r . FremoteName + "/dst" )
2017-01-10 21:47:03 +00:00
require . NoError ( t , err )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2017-01-10 21:47:03 +00:00
require . NoError ( t , err )
2017-01-19 17:26:29 +00:00
// one should be moved to the backup dir and the new one installed
2020-10-04 15:38:29 +00:00
file1 . Path = backupDir + "one" + suffix
2017-01-10 21:47:03 +00:00
file1a . Path = "dst/one"
2017-01-19 17:26:29 +00:00
// two should be unchanged
// three should be moved to the backup dir
2019-03-10 16:50:28 +00:00
if suffixKeepExtension {
2020-10-04 15:38:29 +00:00
file3 . Path = backupDir + "three" + suffix + ".txt"
2019-03-10 16:50:28 +00:00
} else {
2020-10-04 15:38:29 +00:00
file3 . Path = backupDir + "three.txt" + suffix
2019-03-10 16:50:28 +00:00
}
2017-01-10 21:47:03 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 , file1a )
2017-01-10 21:47:03 +00:00
2017-01-19 17:26:29 +00:00
// Now check what happens if we do it again
// Restore a different three and update one in the source
2020-11-05 16:27:01 +00:00
file3a := r . WriteObject ( ctx , "dst/three.txt" , "threeA" , t2 )
2017-01-19 17:26:29 +00:00
file1b := r . WriteFile ( "one" , "oneBB" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 , file1a , file3a )
2017-01-19 17:26:29 +00:00
// This should delete three and overwrite one again, checking
// the files got overwritten correctly in backup-dir
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , fdst , r . Flocal , false )
2017-01-19 17:26:29 +00:00
require . NoError ( t , err )
// one should be moved to the backup dir and the new one installed
2020-10-04 15:38:29 +00:00
file1a . Path = backupDir + "one" + suffix
2017-01-19 17:26:29 +00:00
file1b . Path = "dst/one"
// two should be unchanged
// three should be moved to the backup dir
2019-03-10 16:50:28 +00:00
if suffixKeepExtension {
2020-10-04 15:38:29 +00:00
file3a . Path = backupDir + "three" + suffix + ".txt"
2019-03-10 16:50:28 +00:00
} else {
2020-10-04 15:38:29 +00:00
file3a . Path = backupDir + "three.txt" + suffix
2019-03-10 16:50:28 +00:00
}
2017-01-19 17:26:29 +00:00
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1b , file2 , file3a , file1a )
2017-01-10 21:47:03 +00:00
}
2020-10-04 15:38:29 +00:00
func TestSyncBackupDir ( t * testing . T ) {
testSyncBackupDir ( t , "backup" , "" , false )
}
func TestSyncBackupDirWithSuffix ( t * testing . T ) {
testSyncBackupDir ( t , "backup" , ".bak" , false )
}
2020-02-28 16:56:33 +00:00
func TestSyncBackupDirWithSuffixKeepExtension ( t * testing . T ) {
2020-10-04 15:38:29 +00:00
testSyncBackupDir ( t , "backup" , "-2019-01-01" , true )
}
func TestSyncBackupDirSuffixOnly ( t * testing . T ) {
testSyncBackupDir ( t , "" , ".bak" , false )
2020-02-28 16:56:33 +00:00
}
2017-09-08 15:19:41 +00:00
2019-06-23 03:52:09 +00:00
// Test with Suffix set
func testSyncSuffix ( t * testing . T , suffix string , suffixKeepExtension bool ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2019-06-23 03:52:09 +00:00
r := fstest . NewRun ( t )
if ! operations . CanServerSideMove ( r . Fremote ) {
2020-10-13 21:43:40 +00:00
t . Skip ( "Skipping test as remote does not support server-side move" )
2019-06-23 03:52:09 +00:00
}
2020-11-05 16:27:01 +00:00
r . Mkdir ( ctx , r . Fremote )
2019-06-23 03:52:09 +00:00
2020-11-05 11:33:32 +00:00
ci . Suffix = suffix
ci . SuffixKeepExtension = suffixKeepExtension
2019-06-23 03:52:09 +00:00
// Make the setup so we have one, two, three in the dest
// and one (different), two (same) in the source
2020-11-05 16:27:01 +00:00
file1 := r . WriteObject ( ctx , "dst/one" , "one" , t1 )
file2 := r . WriteObject ( ctx , "dst/two" , "two" , t1 )
file3 := r . WriteObject ( ctx , "dst/three.txt" , "three" , t1 )
2019-06-23 03:52:09 +00:00
file2a := r . WriteFile ( "two" , "two" , t1 )
file1a := r . WriteFile ( "one" , "oneA" , t2 )
file3a := r . WriteFile ( "three.txt" , "threeA" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 )
r . CheckLocalItems ( t , file1a , file2a , file3a )
2019-06-23 03:52:09 +00:00
2020-11-05 16:27:01 +00:00
fdst , err := fs . NewFs ( ctx , r . FremoteName + "/dst" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "one" , "one" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "two" , "two" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "three.txt" , "three.txt" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
// one should be moved to the backup dir and the new one installed
file1 . Path = "dst/one" + suffix
file1a . Path = "dst/one"
// two should be unchanged
// three should be moved to the backup dir
if suffixKeepExtension {
file3 . Path = "dst/three" + suffix + ".txt"
} else {
file3 . Path = "dst/three.txt" + suffix
}
file3a . Path = "dst/three.txt"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 , file1a , file3a )
2019-06-23 03:52:09 +00:00
// Now check what happens if we do it again
// Restore a different three and update one in the source
2019-07-03 16:36:22 +00:00
file3b := r . WriteFile ( "three.txt" , "threeBDifferentSize" , t3 )
2019-06-23 03:52:09 +00:00
file1b := r . WriteFile ( "one" , "oneBB" , t3 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 , file2 , file3 , file1a , file3a )
2019-06-23 03:52:09 +00:00
// This should delete three and overwrite one again, checking
// the files got overwritten correctly in backup-dir
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "one" , "one" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "two" , "two" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
2020-11-05 16:27:01 +00:00
err = operations . CopyFile ( ctx , fdst , r . Flocal , "three.txt" , "three.txt" )
2019-06-23 03:52:09 +00:00
require . NoError ( t , err )
// one should be moved to the backup dir and the new one installed
file1a . Path = "dst/one" + suffix
file1b . Path = "dst/one"
// two should be unchanged
// three should be moved to the backup dir
if suffixKeepExtension {
file3a . Path = "dst/three" + suffix + ".txt"
} else {
file3a . Path = "dst/three.txt" + suffix
}
file3b . Path = "dst/three.txt"
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1b , file3b , file2 , file3a , file1a )
2019-06-23 03:52:09 +00:00
}
func TestSyncSuffix ( t * testing . T ) { testSyncSuffix ( t , ".bak" , false ) }
func TestSyncSuffixKeepExtension ( t * testing . T ) { testSyncSuffix ( t , "-2019-01-01" , true ) }
2017-09-08 15:19:41 +00:00
// Check we can sync two files with differing UTF-8 representations
func TestSyncUTFNorm ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2017-09-17 13:05:33 +00:00
if runtime . GOOS == "darwin" {
t . Skip ( "Can't test UTF normalization on OS X" )
}
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-09-08 15:19:41 +00:00
// Two strings with different unicode normalization (from OS X)
Encoding1 := "Testêé"
Encoding2 := "Testêé"
assert . NotEqual ( t , Encoding1 , Encoding2 )
assert . Equal ( t , norm . NFC . String ( Encoding1 ) , norm . NFC . String ( Encoding2 ) )
file1 := r . WriteFile ( Encoding1 , "This is a test" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2017-09-08 15:19:41 +00:00
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , Encoding2 , "This is a old test" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 )
2017-09-08 15:19:41 +00:00
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
2017-09-08 15:19:41 +00:00
require . NoError ( t , err )
// We should have transferred exactly one file, but kept the
// normalized state of the file.
2019-09-05 20:29:35 +00:00
assert . Equal ( t , toyFileTransfers ( r ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2017-09-08 15:19:41 +00:00
file1 . Path = file2 . Path
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file1 )
2017-09-08 15:19:41 +00:00
}
2017-09-02 08:29:01 +00:00
// Test --immutable
func TestSyncImmutable ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2017-10-29 12:23:10 +00:00
r := fstest . NewRun ( t )
2017-09-02 08:29:01 +00:00
2020-11-05 11:33:32 +00:00
ci . Immutable = true
2017-09-02 08:29:01 +00:00
// Create file on source
file1 := r . WriteFile ( "existing" , "potato" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t )
2017-09-02 08:29:01 +00:00
// Should succeed
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-09-02 08:29:01 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file1 )
2017-09-02 08:29:01 +00:00
// Modify file data and timestamp on source
2017-09-27 15:28:20 +00:00
file2 := r . WriteFile ( "existing" , "tomatoes" , t2 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file1 )
2017-09-02 08:29:01 +00:00
// Should fail with ErrorImmutableModified and not modify local or remote files
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2020-11-05 16:27:01 +00:00
err = Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2017-09-02 08:29:01 +00:00
assert . EqualError ( t , err , fs . ErrorImmutableModified . Error ( ) )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file2 )
r . CheckRemoteItems ( t , file1 )
2017-09-02 08:29:01 +00:00
}
2018-04-24 08:48:35 +00:00
2019-06-03 20:12:10 +00:00
// Test --ignore-case-sync
func TestSyncIgnoreCase ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2019-06-03 20:12:10 +00:00
r := fstest . NewRun ( t )
// Only test if filesystems are case sensitive
if r . Fremote . Features ( ) . CaseInsensitive || r . Flocal . Features ( ) . CaseInsensitive {
t . Skip ( "Skipping test as local or remote are case-insensitive" )
}
2020-11-05 11:33:32 +00:00
ci . IgnoreCaseSync = true
2019-06-03 20:12:10 +00:00
// Create files with different filename casing
file1 := r . WriteFile ( "existing" , "potato" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
2020-11-05 16:27:01 +00:00
file2 := r . WriteObject ( ctx , "EXISTING" , "potato" , t1 )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , file2 )
2019-06-03 20:12:10 +00:00
// Should not copy files that are differently-cased but otherwise identical
2019-07-18 10:13:54 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx)
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
2019-06-03 20:12:10 +00:00
require . NoError ( t , err )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 )
r . CheckRemoteItems ( t , file2 )
2019-06-03 20:12:10 +00:00
}
2023-10-09 02:59:22 +00:00
// Test --fix-case
func TestFixCase ( t * testing . T ) {
ctx := context . Background ( )
ctx , ci := fs . AddConfig ( ctx )
r := fstest . NewRun ( t )
// Only test if remote is case insensitive
if ! r . Fremote . Features ( ) . CaseInsensitive {
t . Skip ( "Skipping test as local or remote are case-sensitive" )
}
ci . FixCase = true
// Create files with different filename casing
file1a := r . WriteFile ( "existing" , "potato" , t1 )
file1b := r . WriteFile ( "existingbutdifferent" , "donut" , t1 )
file1c := r . WriteFile ( "subdira/subdirb/subdirc/hello" , "donut" , t1 )
file1d := r . WriteFile ( "subdira/subdirb/subdirc/subdird/filewithoutcasedifferences" , "donut" , t1 )
r . CheckLocalItems ( t , file1a , file1b , file1c , file1d )
file2a := r . WriteObject ( ctx , "EXISTING" , "potato" , t1 )
file2b := r . WriteObject ( ctx , "EXISTINGBUTDIFFERENT" , "lemonade" , t1 )
file2c := r . WriteObject ( ctx , "SUBDIRA/subdirb/SUBDIRC/HELLO" , "lemonade" , t1 )
file2d := r . WriteObject ( ctx , "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences" , "lemonade" , t1 )
r . CheckRemoteItems ( t , file2a , file2b , file2c , file2d )
// Should force rename of dest file that is differently-cased
accounting . GlobalStats ( ) . ResetCounters ( )
err := Sync ( ctx , r . Fremote , r . Flocal , false )
require . NoError ( t , err )
r . CheckLocalItems ( t , file1a , file1b , file1c , file1d )
r . CheckRemoteItems ( t , file1a , file1b , file1c , file1d )
}
2020-09-09 11:53:21 +00:00
// Test that aborting on --max-transfer works
func TestMaxTransfer ( t * testing . T ) {
2020-11-05 16:27:01 +00:00
ctx := context . Background ( )
2021-02-17 20:21:09 +00:00
ctx , ci := fs . AddConfig ( ctx )
2020-11-05 11:33:32 +00:00
ci . MaxTransfer = 3 * 1024
ci . Transfers = 1
ci . Checkers = 1
ci . CutoffMode = fs . CutoffModeHard
2018-04-24 08:48:35 +00:00
2020-09-09 11:53:21 +00:00
test := func ( t * testing . T , cutoff fs . CutoffMode ) {
r := fstest . NewRun ( t )
2020-11-05 11:33:32 +00:00
ci . CutoffMode = cutoff
2018-04-24 08:48:35 +00:00
2020-09-09 11:53:21 +00:00
if r . Fremote . Name ( ) != "local" {
t . Skip ( "This test only runs on local" )
}
2018-04-24 08:48:35 +00:00
2020-09-09 11:53:21 +00:00
// Create file on source
file1 := r . WriteFile ( "file1" , string ( make ( [ ] byte , 5 * 1024 ) ) , t1 )
file2 := r . WriteFile ( "file2" , string ( make ( [ ] byte , 2 * 1024 ) ) , t1 )
file3 := r . WriteFile ( "file3" , string ( make ( [ ] byte , 3 * 1024 ) ) , t1 )
2021-11-09 11:43:36 +00:00
r . CheckLocalItems ( t , file1 , file2 , file3 )
r . CheckRemoteItems ( t )
2020-09-09 11:53:21 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
2023-10-01 09:02:56 +00:00
// ctx = predictDstFromLogger(ctx) // not currently supported
2020-11-05 16:27:01 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
2020-09-09 11:53:21 +00:00
expectedErr := fserrors . FsError ( accounting . ErrorMaxTransferLimitReachedFatal )
if cutoff != fs . CutoffModeHard {
expectedErr = accounting . ErrorMaxTransferLimitReachedGraceful
}
fserrors . Count ( expectedErr )
assert . Equal ( t , expectedErr , err )
}
t . Run ( "Hard" , func ( t * testing . T ) { test ( t , fs . CutoffModeHard ) } )
t . Run ( "Soft" , func ( t * testing . T ) { test ( t , fs . CutoffModeSoft ) } )
t . Run ( "Cautious" , func ( t * testing . T ) { test ( t , fs . CutoffModeCautious ) } )
2018-04-24 08:48:35 +00:00
}
2021-07-07 13:50:19 +00:00
func testSyncConcurrent ( t * testing . T , subtest string ) {
const (
NFILES = 20
NCHECKERS = 4
NTRANSFERS = 4
)
ctx , ci := fs . AddConfig ( context . Background ( ) )
ci . Checkers = NCHECKERS
ci . Transfers = NTRANSFERS
r := fstest . NewRun ( t )
stats := accounting . GlobalStats ( )
itemsBefore := [ ] fstest . Item { }
itemsAfter := [ ] fstest . Item { }
for i := 0 ; i < NFILES ; i ++ {
nameBoth := fmt . Sprintf ( "both%d" , i )
nameOnly := fmt . Sprintf ( "only%d" , i )
switch subtest {
case "delete" :
fileBoth := r . WriteBoth ( ctx , nameBoth , "potato" , t1 )
fileOnly := r . WriteObject ( ctx , nameOnly , "potato" , t1 )
itemsBefore = append ( itemsBefore , fileBoth , fileOnly )
itemsAfter = append ( itemsAfter , fileBoth )
case "truncate" :
fileBoth := r . WriteBoth ( ctx , nameBoth , "potato" , t1 )
fileFull := r . WriteObject ( ctx , nameOnly , "potato" , t1 )
fileEmpty := r . WriteFile ( nameOnly , "" , t1 )
itemsBefore = append ( itemsBefore , fileBoth , fileFull )
itemsAfter = append ( itemsAfter , fileBoth , fileEmpty )
}
}
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , itemsBefore ... )
2021-07-07 13:50:19 +00:00
stats . ResetErrors ( )
2023-10-01 09:02:56 +00:00
ctx = predictDstFromLogger ( ctx )
2021-07-07 13:50:19 +00:00
err := Sync ( ctx , r . Fremote , r . Flocal , false )
2023-10-01 09:02:56 +00:00
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
2021-11-04 10:12:57 +00:00
if errors . Is ( err , fs . ErrorCantUploadEmptyFiles ) {
2021-10-23 10:35:20 +00:00
t . Skipf ( "Skip test because remote cannot upload empty files" )
}
2021-07-07 13:50:19 +00:00
assert . NoError ( t , err , "Sync must not return a error" )
assert . False ( t , stats . Errored ( ) , "Low level errors must not have happened" )
2021-11-09 11:43:36 +00:00
r . CheckRemoteItems ( t , itemsAfter ... )
2021-07-07 13:50:19 +00:00
}
func TestSyncConcurrentDelete ( t * testing . T ) {
testSyncConcurrent ( t , "delete" )
}
func TestSyncConcurrentTruncate ( t * testing . T ) {
testSyncConcurrent ( t , "truncate" )
}
2023-10-01 09:02:56 +00:00
2024-02-29 00:29:38 +00:00
// Tests that nothing is transferred when src and dst already match
// Run the same sync twice, ensure no action is taken the second time
2024-04-04 17:03:20 +00:00
func testNothingToTransfer ( t * testing . T , copyEmptySrcDirs bool ) {
2024-02-29 00:29:38 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
ctx , _ := fs . AddConfig ( context . Background ( ) )
r := fstest . NewRun ( t )
file1 := r . WriteFile ( "sub dir/hello world" , "hello world" , t1 )
file2 := r . WriteFile ( "sub dir2/very/very/very/very/very/nested/subdir/hello world" , "hello world" , t1 )
r . CheckLocalItems ( t , file1 , file2 )
_ , err := operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir" , t2 )
if err != nil && ! errors . Is ( err , fs . ErrorNotImplemented ) {
require . NoError ( t , err )
}
r . Mkdir ( ctx , r . Fremote )
_ , err = operations . MkdirModTime ( ctx , r . Fremote , "sub dir" , t3 )
require . NoError ( t , err )
// set logging
// (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken)
oldLogLevel := fs . GetConfig ( context . Background ( ) ) . LogLevel
defer func ( ) { fs . GetConfig ( context . Background ( ) ) . LogLevel = oldLogLevel } ( ) // reset to old val after test
// need to do this as fs.Infof only respects the globalConfig
fs . GetConfig ( context . Background ( ) ) . LogLevel = fs . LogLevelInfo
accounting . GlobalStats ( ) . ResetCounters ( )
ctx = predictDstFromLogger ( ctx )
output := bilib . CaptureOutput ( func ( ) {
2024-04-04 17:03:20 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , copyEmptySrcDirs )
2024-02-29 00:29:38 +00:00
require . NoError ( t , err )
} )
require . NotNil ( t , output )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file1 , file2 )
// Check that the modtimes of the directories are as expected
2024-04-17 15:55:17 +00:00
r . CheckDirectoryModTimes ( t , "sub dir" , "sub dir2" , "sub dir2/very" , "sub dir2/very/very" , "sub dir2/very/very/very/very/very/nested/subdir" )
2024-02-29 00:29:38 +00:00
// check that actions were taken
assert . True ( t , strings . Contains ( string ( output ) , "Copied" ) , ` expected to find at least one "Copied" log: ` + string ( output ) )
if r . Fremote . Features ( ) . DirSetModTime != nil || r . Fremote . Features ( ) . MkdirMetadata != nil {
assert . True ( t , strings . Contains ( string ( output ) , "Set directory modification time" ) , ` expected to find at least one "Set directory modification time" log: ` + string ( output ) )
}
assert . False ( t , strings . Contains ( string ( output ) , "There was nothing to transfer" ) , ` expected to find no "There was nothing to transfer" logs, but found one: ` + string ( output ) )
2024-03-08 14:45:48 +00:00
assert . True ( t , accounting . GlobalStats ( ) . GetTransfers ( ) >= 2 )
2024-02-29 00:29:38 +00:00
// run it again and make sure no actions were taken
accounting . GlobalStats ( ) . ResetCounters ( )
ctx = predictDstFromLogger ( ctx )
output = bilib . CaptureOutput ( func ( ) {
2024-04-04 17:03:20 +00:00
err = CopyDir ( ctx , r . Fremote , r . Flocal , copyEmptySrcDirs )
2024-02-29 00:29:38 +00:00
require . NoError ( t , err )
} )
require . NotNil ( t , output )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckLocalItems ( t , file1 , file2 )
r . CheckRemoteItems ( t , file1 , file2 )
// Check that the modtimes of the directories are as expected
2024-04-17 15:55:17 +00:00
r . CheckDirectoryModTimes ( t , "sub dir" , "sub dir2" , "sub dir2/very" , "sub dir2/very/very" , "sub dir2/very/very/very/very/very/nested/subdir" )
2024-02-29 00:29:38 +00:00
// check that actions were NOT taken
assert . False ( t , strings . Contains ( string ( output ) , "Copied" ) , ` expected to find no "Copied" logs, but found one: ` + string ( output ) )
if r . Fremote . Features ( ) . DirSetModTime != nil || r . Fremote . Features ( ) . MkdirMetadata != nil {
assert . False ( t , strings . Contains ( string ( output ) , "Set directory modification time" ) , ` expected to find no "Set directory modification time" logs, but found one: ` + string ( output ) )
2024-04-04 17:03:20 +00:00
assert . False ( t , strings . Contains ( string ( output ) , "Updated directory metadata" ) , ` expected to find no "Updated directory metadata" logs, but found one: ` + string ( output ) )
assert . False ( t , strings . Contains ( string ( output ) , "directory" ) , ` expected to find no "directory"-related logs, but found one: ` + string ( output ) ) // catch-all
2024-02-29 00:29:38 +00:00
}
assert . True ( t , strings . Contains ( string ( output ) , "There was nothing to transfer" ) , ` expected to find a "There was nothing to transfer" log: ` + string ( output ) )
assert . Equal ( t , int64 ( 0 ) , accounting . GlobalStats ( ) . GetTransfers ( ) )
2024-04-16 18:39:30 +00:00
2024-04-17 15:55:17 +00:00
// check nested empty dir behavior (FIXME: probably belongs in a separate test)
2024-04-16 18:39:30 +00:00
if r . Fremote . Features ( ) . DirSetModTime == nil && r . Fremote . Features ( ) . MkdirMetadata == nil {
return
}
file3 := r . WriteFile ( "sub dir2/sub dir3/hello world" , "hello again, world" , t1 )
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dir2" , t1 )
assert . NoError ( t , err )
_ , err = operations . SetDirModTime ( ctx , r . Fremote , nil , "sub dir2" , t1 )
assert . NoError ( t , err )
2024-04-17 15:55:17 +00:00
_ , err = operations . MkdirModTime ( ctx , r . Flocal , "sub dirEmpty/sub dirEmpty2" , t2 )
assert . NoError ( t , err )
_ , err = operations . SetDirModTime ( ctx , r . Flocal , nil , "sub dirEmpty" , t2 )
assert . NoError ( t , err )
2024-04-16 18:39:30 +00:00
accounting . GlobalStats ( ) . ResetCounters ( )
ctx = predictDstFromLogger ( ctx )
output = bilib . CaptureOutput ( func ( ) {
err = CopyDir ( ctx , r . Fremote , r . Flocal , copyEmptySrcDirs )
require . NoError ( t , err )
} )
require . NotNil ( t , output )
testLoggerVsLsf ( ctx , r . Fremote , operations . GetLoggerOpt ( ctx ) . JSON , t )
r . CheckLocalItems ( t , file1 , file2 , file3 )
r . CheckRemoteItems ( t , file1 , file2 , file3 )
2024-04-17 15:55:17 +00:00
// Check that the modtimes of the directories are as expected
r . CheckDirectoryModTimes ( t , "sub dir" , "sub dir2" , "sub dir2/very" , "sub dir2/very/very" , "sub dir2/very/very/very/very/very/nested/subdir" , "sub dir2/sub dir3" )
if copyEmptySrcDirs {
r . CheckDirectoryModTimes ( t , "sub dirEmpty" , "sub dirEmpty/sub dirEmpty2" )
assert . True ( t , strings . Contains ( string ( output ) , "sub dirEmpty:" ) , ` expected to find at least one "sub dirEmpty:" log: ` + string ( output ) )
} else {
assert . False ( t , strings . Contains ( string ( output ) , "sub dirEmpty:" ) , ` expected to find no "sub dirEmpty:" logs, but found one (empty dir was synced and shouldn't have been): ` + string ( output ) )
}
2024-04-16 18:39:30 +00:00
assert . True ( t , strings . Contains ( string ( output ) , "sub dir3:" ) , ` expected to find at least one "sub dir3:" log: ` + string ( output ) )
2024-04-17 15:55:17 +00:00
assert . False ( t , strings . Contains ( string ( output ) , "sub dir2/very:" ) , ` expected to find no "sub dir2/very:" logs, but found one (unmodified dir was marked modified): ` + string ( output ) )
2024-02-29 00:29:38 +00:00
}
2024-04-04 17:03:20 +00:00
func TestNothingToTransferWithEmptyDirs ( t * testing . T ) {
testNothingToTransfer ( t , true )
}
func TestNothingToTransferWithoutEmptyDirs ( t * testing . T ) {
testNothingToTransfer ( t , false )
}
2023-10-01 09:02:56 +00:00
// for testing logger:
func predictDstFromLogger ( ctx context . Context ) context . Context {
opt := operations . NewLoggerOpt ( )
var lock mutex . Mutex
opt . LoggerFn = func ( ctx context . Context , sigil operations . Sigil , src , dst fs . DirEntry , err error ) {
lock . Lock ( )
defer lock . Unlock ( )
// ignore dirs for our purposes here
if err == fs . ErrorIsDir {
return
}
winner := operations . WinningSide ( ctx , sigil , src , dst , err )
if winner . Obj != nil {
file := winner . Obj
obj , ok := file . ( fs . ObjectInfo )
checksum := ""
2023-11-06 07:45:51 +00:00
timeFormat := "2006-01-02 15:04:05"
2023-10-01 09:02:56 +00:00
if ok {
if obj . Fs ( ) . Hashes ( ) . GetOne ( ) == hash . MD5 {
// skip if no MD5
2023-11-06 07:45:51 +00:00
checksum , _ = obj . Hash ( ctx , hash . MD5 )
2023-10-01 09:02:56 +00:00
}
2023-11-06 07:45:51 +00:00
timeFormat = operations . FormatForLSFPrecision ( obj . Fs ( ) . Precision ( ) )
2023-10-01 09:02:56 +00:00
}
errMsg := ""
if winner . Err != nil {
errMsg = ";" + winner . Err . Error ( )
}
2023-11-06 07:45:51 +00:00
operations . SyncFprintf ( opt . JSON , "%s;%s;%v;%s%s\n" , file . ModTime ( ctx ) . Local ( ) . Format ( timeFormat ) , checksum , file . Size ( ) , file . Remote ( ) , errMsg )
2023-10-01 09:02:56 +00:00
}
}
return operations . WithSyncLogger ( ctx , opt )
}
func DstLsf ( ctx context . Context , Fremote fs . Fs ) * bytes . Buffer {
var opt = operations . ListJSONOpt {
NoModTime : false ,
NoMimeType : true ,
DirsOnly : false ,
FilesOnly : true ,
Recurse : true ,
ShowHash : true ,
HashTypes : [ ] string { "MD5" } ,
}
var list operations . ListFormat
list . SetSeparator ( ";" )
timeFormat := operations . FormatForLSFPrecision ( Fremote . Precision ( ) )
list . AddModTime ( timeFormat )
list . AddHash ( hash . MD5 )
list . AddSize ( )
list . AddPath ( )
out := new ( bytes . Buffer )
err := operations . ListJSON ( ctx , Fremote , "" , & opt , func ( item * operations . ListJSONItem ) error {
_ , _ = fmt . Fprintln ( out , list . Format ( item ) )
return nil
} )
if err != nil {
fs . Errorf ( Fremote , "ListJSON error: %v" , err )
}
return out
}
func LoggerMatchesLsf ( logger , lsf * bytes . Buffer ) error {
loggerSplit := bytes . Split ( logger . Bytes ( ) , [ ] byte ( "\n" ) )
sort . SliceStable ( loggerSplit , func ( i int , j int ) bool { return string ( loggerSplit [ i ] ) < string ( loggerSplit [ j ] ) } )
lsfSplit := bytes . Split ( lsf . Bytes ( ) , [ ] byte ( "\n" ) )
sort . SliceStable ( lsfSplit , func ( i int , j int ) bool { return string ( lsfSplit [ i ] ) < string ( lsfSplit [ j ] ) } )
loggerJoined := bytes . Join ( loggerSplit , [ ] byte ( "\n" ) )
lsfJoined := bytes . Join ( lsfSplit , [ ] byte ( "\n" ) )
if bytes . Equal ( loggerJoined , lsfJoined ) {
return nil
}
Diff ( string ( loggerJoined ) , string ( lsfJoined ) )
return fmt . Errorf ( "logger does not match lsf! \nlogger: \n%s \nlsf: \n%s" , loggerJoined , lsfJoined )
}
func Diff ( rev1 , rev2 string ) {
fmt . Printf ( "Diff of %q and %q\n" , "logger" , "lsf" )
cmd := exec . Command ( "bash" , "-c" , fmt . Sprintf ( ` diff <(echo "%s") <(echo "%s") ` , rev1 , rev2 ) )
out , _ := cmd . Output ( )
_ , _ = os . Stdout . Write ( out )
}
func testLoggerVsLsf ( ctx context . Context , Fremote fs . Fs , logger * bytes . Buffer , t * testing . T ) {
var newlogger bytes . Buffer
canTestModtime := fs . GetModifyWindow ( ctx , Fremote ) != fs . ModTimeNotSupported
canTestHash := Fremote . Hashes ( ) . Contains ( hash . MD5 )
if ! canTestHash || ! canTestModtime {
loggerSplit := bytes . Split ( logger . Bytes ( ) , [ ] byte ( "\n" ) )
for i , line := range loggerSplit {
elements := bytes . Split ( line , [ ] byte ( ";" ) )
if len ( elements ) >= 2 {
if ! canTestModtime {
elements [ 0 ] = [ ] byte ( "" )
}
if ! canTestHash {
elements [ 1 ] = [ ] byte ( "" )
}
}
loggerSplit [ i ] = bytes . Join ( elements , [ ] byte ( ";" ) )
}
newlogger . Write ( bytes . Join ( loggerSplit , [ ] byte ( "\n" ) ) )
} else {
newlogger . Write ( logger . Bytes ( ) )
}
r := fstest . NewRun ( t )
if r . Flocal . Precision ( ) == Fremote . Precision ( ) && r . Flocal . Hashes ( ) . Contains ( hash . MD5 ) && canTestHash {
2023-11-06 07:45:51 +00:00
lsf := DstLsf ( ctx , Fremote )
err := LoggerMatchesLsf ( & newlogger , lsf )
2023-10-01 09:02:56 +00:00
require . NoError ( t , err )
}
}