fstests: add integration tests for Directory Metadata and ModTime
This commit is contained in:
Nick Craig-Wood
parent
fd1ca2dfe8
commit
61d76ae47d
3 changed files with 378 additions and 15 deletions
|
|
@ -1639,3 +1639,112 @@ func TestTouchDir(t *testing.T) {
|
|||
r.CheckRemoteItems(t, file1, file2, file3)
|
||||
}
|
||||
}
|
||||
|
||||
var testMetadata = fs.Metadata{
|
||||
// System metadata supported by all backends
|
||||
"mtime": t1.Format(time.RFC3339Nano),
|
||||
// User metadata
|
||||
"potato": "jersey",
|
||||
}
|
||||
|
||||
func TestMkdirMetadata(t *testing.T) {
|
||||
const name = "dir with metadata"
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
r := fstest.NewRun(t)
|
||||
features := r.Fremote.Features()
|
||||
if features.MkdirMetadata == nil {
|
||||
t.Skip("Skipping test as remote does not support MkdirMetadata")
|
||||
}
|
||||
|
||||
newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, newDst)
|
||||
|
||||
require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata)
|
||||
}
|
||||
|
||||
func TestMkdirModTime(t *testing.T) {
|
||||
const name = "directory with modtime"
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
|
||||
t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")
|
||||
}
|
||||
newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
|
||||
fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
|
||||
}
|
||||
|
||||
func TestCopyDirMetadata(t *testing.T) {
|
||||
const nameNonExistent = "non existent directory"
|
||||
const nameExistent = "existing directory"
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
r := fstest.NewRun(t)
|
||||
if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
|
||||
t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
|
||||
}
|
||||
|
||||
// Create a source local directory with metadata
|
||||
newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, newSrc)
|
||||
|
||||
// First try with the directory not existing
|
||||
newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, newDst)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata)
|
||||
|
||||
// Then try with the directory existing
|
||||
require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent))
|
||||
require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent))
|
||||
existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent)
|
||||
|
||||
newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, newDst)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
|
||||
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata)
|
||||
}
|
||||
|
||||
func TestSetDirModTime(t *testing.T) {
|
||||
const name = "set modtime on existing directory"
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime {
|
||||
t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime")
|
||||
}
|
||||
|
||||
// First try with the directory not existing - should return an error
|
||||
newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, newDst)
|
||||
|
||||
// Then try with the directory existing
|
||||
require.NoError(t, r.Fremote.Mkdir(ctx, name))
|
||||
existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
|
||||
|
||||
newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, newDst)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
|
||||
fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -519,3 +519,71 @@ func Purge(f fs.Fs) {
|
|||
log.Printf("purge failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// NewDirectory finds the directory with remote in f
|
||||
//
|
||||
// One day this will be an rclone primitive
|
||||
func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory {
|
||||
var err error
|
||||
var dir fs.Directory
|
||||
sleepTime := 1 * time.Second
|
||||
root := path.Dir(remote)
|
||||
if root == "." {
|
||||
root = ""
|
||||
}
|
||||
for i := 1; i <= *ListRetries; i++ {
|
||||
var entries fs.DirEntries
|
||||
entries, err = f.List(ctx, root)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
var ok bool
|
||||
dir, ok = entry.(fs.Directory)
|
||||
if ok && dir.Remote() == remote {
|
||||
return dir
|
||||
}
|
||||
}
|
||||
err = fmt.Errorf("directory %q not found in %q", remote, root)
|
||||
t.Logf("Sleeping for %v for findDir eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err)
|
||||
time.Sleep(sleepTime)
|
||||
sleepTime = (sleepTime * 3) / 2
|
||||
}
|
||||
require.NoError(t, err)
|
||||
return dir
|
||||
}
|
||||
|
||||
// CheckEntryMetadata checks the metadata on the directory
|
||||
//
|
||||
// This checks a limited set of metadata on the directory
|
||||
func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) {
|
||||
features := f.Features()
|
||||
do, ok := entry.(fs.Metadataer)
|
||||
require.True(t, ok, "Didn't find expected Metadata() method on %T", entry)
|
||||
gotMeta, err := do.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
for k, v := range wantMeta {
|
||||
switch k {
|
||||
case "mtime", "atime", "btime", "ctime":
|
||||
// Check the system time Metadata
|
||||
wantT, err := time.Parse(time.RFC3339, v)
|
||||
require.NoError(t, err)
|
||||
gotT, err := time.Parse(time.RFC3339, gotMeta[k])
|
||||
require.NoError(t, err)
|
||||
AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision())
|
||||
default:
|
||||
// Check the User metadata if we can
|
||||
_, isDir := entry.(fs.Directory)
|
||||
if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) {
|
||||
assert.Equal(t, v, gotMeta[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CheckDirModTime checks the modtime on the directory
|
||||
func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) {
|
||||
gotT := dir.ModTime(ctx)
|
||||
AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -314,10 +314,12 @@ type Opt struct {
|
|||
SkipFsMatch bool // if set skip exact matching of Fs value
|
||||
TiersToTest []string // List of tiers which can be tested in setTier test
|
||||
ChunkedUpload ChunkedUploadConfig
|
||||
UnimplementableFsMethods []string // List of methods which can't be implemented in this wrapping Fs
|
||||
UnimplementableObjectMethods []string // List of methods which can't be implemented in this wrapping Fs
|
||||
UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs
|
||||
UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs
|
||||
UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs
|
||||
SkipFsCheckWrap bool // if set skip FsCheckWrap
|
||||
SkipObjectCheckWrap bool // if set skip ObjectCheckWrap
|
||||
SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap
|
||||
SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks
|
||||
QuickTestOK bool // if set, run this test with make quicktest
|
||||
}
|
||||
|
|
@ -1513,8 +1515,8 @@ func Run(t *testing.T, opt *Opt) {
|
|||
}
|
||||
}
|
||||
if !features.ReadMetadata {
|
||||
if metadata != nil {
|
||||
require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata")
|
||||
if metadata != nil && !features.Overlay {
|
||||
require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata)
|
||||
}
|
||||
} else if features.WriteMetadata {
|
||||
require.NotNil(t, metadata)
|
||||
|
|
@ -2301,6 +2303,190 @@ func Run(t *testing.T, opt *Opt) {
|
|||
// somehow confused about root and absolute root.
|
||||
})
|
||||
|
||||
// FsDirSetModTime tests setting the mod time on a directory if possible
|
||||
t.Run("FsDirSetModTime", func(t *testing.T) {
|
||||
const name = "dir-mod-time"
|
||||
do := f.Features().DirSetModTime
|
||||
if do == nil {
|
||||
t.Skip("FS has no DirSetModTime interface")
|
||||
}
|
||||
|
||||
// Set ModTime on non existing directory should return error
|
||||
t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
err := do(ctx, name, t1)
|
||||
require.Error(t, err)
|
||||
|
||||
// Make the directory and try again
|
||||
err = f.Mkdir(ctx, name)
|
||||
require.NoError(t, err)
|
||||
err = do(ctx, name, t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check the modtime got set properly
|
||||
dir := fstest.NewDirectory(ctx, t, f, name)
|
||||
fstest.CheckDirModTime(ctx, t, f, dir, t1)
|
||||
|
||||
// Tidy up
|
||||
err = f.Rmdir(ctx, name)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
var testMetadata = fs.Metadata{
|
||||
// System metadata supported by all backends
|
||||
"mtime": "2001-02-03T04:05:06.499999999Z",
|
||||
// User metadata
|
||||
"potato": "jersey",
|
||||
}
|
||||
var testMetadata2 = fs.Metadata{
|
||||
// System metadata supported by all backends
|
||||
"mtime": "2002-02-03T04:05:06.499999999Z",
|
||||
// User metadata
|
||||
"potato": "king edwards",
|
||||
}
|
||||
|
||||
// FsMkdirMetadata tests creating a directory with metadata if possible
|
||||
t.Run("FsMkdirMetadata", func(t *testing.T) {
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
const name = "dir-metadata"
|
||||
do := f.Features().MkdirMetadata
|
||||
if do == nil {
|
||||
t.Skip("FS has no MkdirMetadata interface")
|
||||
}
|
||||
assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
|
||||
|
||||
// Create the directory from fresh
|
||||
dir, err := do(ctx, name, testMetadata)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dir)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
|
||||
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
|
||||
|
||||
// Now update the metadata on the existing directory
|
||||
t.Run("Update", func(t *testing.T) {
|
||||
dir, err := do(ctx, name, testMetadata2)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dir)
|
||||
|
||||
// Check the returned directory and one read from the listing
|
||||
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2)
|
||||
// The TestUnionPolicy2 has randomness in it so it sets metadata on
|
||||
// one directory but can read a different one from the listing.
|
||||
if f.Name() != "TestUnionPolicy2" {
|
||||
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2)
|
||||
}
|
||||
})
|
||||
|
||||
// Now test the Directory methods
|
||||
t.Run("CheckDirectory", func(t *testing.T) {
|
||||
_, ok := dir.(fs.Object)
|
||||
assert.False(t, ok, "Directory must not type assert to Object")
|
||||
_, ok = dir.(fs.ObjectInfo)
|
||||
assert.False(t, ok, "Directory must not type assert to ObjectInfo")
|
||||
})
|
||||
|
||||
// Tidy up
|
||||
err = f.Rmdir(ctx, name)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
// FsDirectory checks methods on the directory object
|
||||
t.Run("FsDirectory", func(t *testing.T) {
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
const name = "dir-methods"
|
||||
features := f.Features()
|
||||
|
||||
if !features.CanHaveEmptyDirectories {
|
||||
t.Skip("Can't test if can't have empty directories")
|
||||
}
|
||||
if !features.ReadDirMetadata &&
|
||||
!features.WriteDirMetadata &&
|
||||
!features.WriteDirSetModTime &&
|
||||
!features.UserDirMetadata &&
|
||||
!features.Overlay &&
|
||||
features.UnWrap == nil {
|
||||
t.Skip("FS has no Directory methods and doesn't Wrap")
|
||||
}
|
||||
|
||||
// Create a directory to start with
|
||||
err := f.Mkdir(ctx, name)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the directory object
|
||||
dir := fstest.NewDirectory(ctx, t, f, name)
|
||||
_, ok := dir.(fs.Object)
|
||||
assert.False(t, ok, "Directory must not type assert to Object")
|
||||
_, ok = dir.(fs.ObjectInfo)
|
||||
assert.False(t, ok, "Directory must not type assert to ObjectInfo")
|
||||
|
||||
// Now test the directory methods
|
||||
t.Run("ReadDirMetadata", func(t *testing.T) {
|
||||
if !features.ReadDirMetadata {
|
||||
t.Skip("Directories don't support ReadDirMetadata")
|
||||
}
|
||||
if f.Name() == "TestUnionPolicy3" {
|
||||
t.Skipf("Test unreliable on %q", f.Name())
|
||||
}
|
||||
fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{
|
||||
"mtime": dir.ModTime(ctx).Format(time.RFC3339Nano),
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("WriteDirMetadata", func(t *testing.T) {
|
||||
if !features.WriteDirMetadata {
|
||||
t.Skip("Directories don't support WriteDirMetadata")
|
||||
}
|
||||
assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
|
||||
do, ok := dir.(fs.SetMetadataer)
|
||||
require.True(t, ok, "Expected to find SetMetadata method on Directory")
|
||||
err := do.SetMetadata(ctx, testMetadata)
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
|
||||
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
|
||||
})
|
||||
|
||||
t.Run("WriteDirSetModTime", func(t *testing.T) {
|
||||
if !features.WriteDirSetModTime {
|
||||
t.Skip("Directories don't support WriteDirSetModTime")
|
||||
}
|
||||
assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime")
|
||||
|
||||
t1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
|
||||
|
||||
do, ok := dir.(fs.SetModTimer)
|
||||
require.True(t, ok, "Expected to find SetMetadata method on Directory")
|
||||
err := do.SetModTime(ctx, t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CheckDirModTime(ctx, t, f, dir, t1)
|
||||
fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1)
|
||||
})
|
||||
|
||||
// Check to see if Fs that wrap other Directories implement all the optional methods
|
||||
t.Run("DirectoryCheckWrap", func(t *testing.T) {
|
||||
if opt.SkipDirectoryCheckWrap {
|
||||
t.Skip("Skipping DirectoryCheckWrap on this Fs")
|
||||
}
|
||||
if !features.Overlay && features.UnWrap == nil {
|
||||
t.Skip("Not a wrapping Fs")
|
||||
}
|
||||
_, unsupported := fs.DirectoryOptionalInterfaces(dir)
|
||||
for _, name := range unsupported {
|
||||
if !stringsContains(name, opt.UnimplementableDirectoryMethods) {
|
||||
t.Errorf("Missing Directory wrapper for %s", name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Tidy up
|
||||
err = f.Rmdir(ctx, name)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
// Purge the folder
|
||||
err = operations.Purge(ctx, f, "")
|
||||
if !errors.Is(err, fs.ErrorDirNotFound) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue