2017-10-28 19:01:34 +00:00
package vfs
2017-05-02 21:35:07 +00:00
import (
2019-06-17 08:34:30 +00:00
"context"
2021-03-16 11:19:18 +00:00
"fmt"
2017-10-25 09:00:26 +00:00
"os"
2017-05-02 21:35:07 +00:00
"path"
2017-10-27 21:07:59 +00:00
"sort"
2017-05-07 12:04:20 +00:00
"strings"
2017-05-02 21:35:07 +00:00
"sync"
2020-07-08 09:27:26 +00:00
"sync/atomic"
2017-05-02 21:35:07 +00:00
"time"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/dirtree"
"github.com/rclone/rclone/fs/list"
2019-10-17 13:41:55 +00:00
"github.com/rclone/rclone/fs/log"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
2020-02-28 14:44:15 +00:00
"github.com/rclone/rclone/vfs/vfscommon"
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
"golang.org/x/text/unicode/norm"
2017-05-02 21:35:07 +00:00
)
// Dir represents a directory entry
type Dir struct {
2022-11-10 11:34:57 +00:00
vfs * VFS // read only
inode uint64 // read only: inode number
f fs . Fs // read only
cleanupTimer * time . Timer // read only: timer to call cacheCleanup
2019-11-06 21:48:43 +00:00
mu sync . RWMutex // protects the following
parent * Dir // parent, nil for root
2017-05-02 21:35:07 +00:00
path string
2017-10-26 15:05:34 +00:00
entry fs . Directory
2020-06-23 09:59:10 +00:00
read time . Time // time directory entry last read
items map [ string ] Node // directory entries - can be empty but not nil
virtual map [ string ] vState // virtual directory entries - may be nil
2020-07-08 09:27:26 +00:00
sys atomic . Value // user defined info to be attached here
2020-07-08 09:27:26 +00:00
modTimeMu sync . Mutex // protects the following
modTime time . Time
2022-11-14 07:37:12 +00:00
_hasVirtual atomic . Bool // shows if the directory has virtual entries
2017-05-02 21:35:07 +00:00
}
2020-06-23 09:59:10 +00:00
//go:generate stringer -type=vState
// vState describes the state of the virtual directory entries
type vState byte
const (
2020-11-10 12:58:03 +00:00
vOK vState = iota // Not virtual
vAddFile // added file
vAddDir // added directory
vDel // removed file or directory
2020-06-23 09:59:10 +00:00
)
2017-10-28 19:01:34 +00:00
func newDir ( vfs * VFS , f fs . Fs , parent * Dir , fsDir fs . Directory ) * Dir {
2022-11-10 11:34:57 +00:00
d := & Dir {
2017-10-28 19:01:34 +00:00
vfs : vfs ,
2017-05-02 21:35:07 +00:00
f : f ,
2017-10-26 15:05:34 +00:00
parent : parent ,
entry : fsDir ,
2017-06-30 12:37:29 +00:00
path : fsDir . Remote ( ) ,
2019-06-17 08:34:30 +00:00
modTime : fsDir . ModTime ( context . TODO ( ) ) ,
2017-10-29 17:37:54 +00:00
inode : newInode ( ) ,
2018-03-01 15:51:05 +00:00
items : make ( map [ string ] Node ) ,
2017-05-02 21:35:07 +00:00
}
2022-11-10 11:34:57 +00:00
d . cleanupTimer = time . AfterFunc ( vfs . Opt . DirCacheTime * 2 , d . cacheCleanup )
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( false )
2022-11-10 11:34:57 +00:00
return d
}
func ( d * Dir ) cacheCleanup ( ) {
defer func ( ) {
// We should never panic here
_ = recover ( )
} ( )
when := time . Now ( )
d . mu . Lock ( )
_ , stale := d . _age ( when )
d . mu . Unlock ( )
if stale {
d . ForgetAll ( )
}
2017-05-02 21:35:07 +00:00
}
2020-06-23 09:59:10 +00:00
// String converts it to printable
2017-05-09 10:29:02 +00:00
func ( d * Dir ) String ( ) string {
if d == nil {
return "<nil *Dir>"
}
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
defer d . mu . RUnlock ( )
2017-05-09 10:29:02 +00:00
return d . path + "/"
}
2021-03-16 11:19:18 +00:00
// Dumps the directory tree to the string builder with the given indent
2023-03-25 22:30:15 +00:00
//
//lint:ignore U1000 false positive when running staticcheck,
//nolint:unused // Don't include unused when running golangci-lint
2021-03-16 11:19:18 +00:00
func ( d * Dir ) dumpIndent ( out * strings . Builder , indent string ) {
if d == nil {
fmt . Fprintf ( out , "%s<nil *Dir>\n" , indent )
return
}
d . mu . RLock ( )
defer d . mu . RUnlock ( )
fmt . Fprintf ( out , "%sPath: %s\n" , indent , d . path )
fmt . Fprintf ( out , "%sEntry: %v\n" , indent , d . entry )
fmt . Fprintf ( out , "%sRead: %v\n" , indent , d . read )
fmt . Fprintf ( out , "%s- items %d\n" , indent , len ( d . items ) )
// Sort?
for leaf , node := range d . items {
switch x := node . ( type ) {
case * Dir :
fmt . Fprintf ( out , "%s %s/ - %v\n" , indent , leaf , x )
// check the parent is correct
if x . parent != d {
fmt . Fprintf ( out , "%s PARENT POINTER WRONG\n" , indent )
}
x . dumpIndent ( out , indent + "\t" )
case * File :
fmt . Fprintf ( out , "%s %s - %v\n" , indent , leaf , x )
default :
panic ( "bad dir entry" )
}
}
fmt . Fprintf ( out , "%s- virtual %d\n" , indent , len ( d . virtual ) )
for leaf , state := range d . virtual {
fmt . Fprintf ( out , "%s %s - %v\n" , indent , leaf , state )
}
}
// Dumps a nicely formatted directory tree to a string
2023-03-25 22:30:15 +00:00
//
//lint:ignore U1000 false positive when running staticcheck,
//nolint:unused // Don't include unused when running golangci-lint
2021-03-16 11:19:18 +00:00
func ( d * Dir ) dump ( ) string {
var out strings . Builder
d . dumpIndent ( & out , "" )
return out . String ( )
}
2017-05-02 21:35:07 +00:00
// IsFile returns false for Dir - satisfies Node interface
func ( d * Dir ) IsFile ( ) bool {
return false
}
2017-10-25 09:00:26 +00:00
// IsDir returns true for Dir - satisfies Node interface
func ( d * Dir ) IsDir ( ) bool {
return true
}
// Mode bits of the directory - satisfies Node interface
func ( d * Dir ) Mode ( ) ( mode os . FileMode ) {
2017-10-29 21:14:05 +00:00
return d . vfs . Opt . DirPerms
2017-10-25 09:00:26 +00:00
}
// Name (base) of the directory - satisfies Node interface
func ( d * Dir ) Name ( ) ( name string ) {
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
2017-10-25 09:00:26 +00:00
name = path . Base ( d . path )
2019-11-06 21:48:43 +00:00
d . mu . RUnlock ( )
2017-10-25 09:00:26 +00:00
if name == "." {
name = "/"
}
return name
}
2017-11-18 11:47:21 +00:00
// Path of the directory - satisfies Node interface
func ( d * Dir ) Path ( ) ( name string ) {
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
defer d . mu . RUnlock ( )
2017-11-18 11:47:21 +00:00
return d . path
}
2017-10-25 09:00:26 +00:00
// Sys returns underlying data source (can be nil) - satisfies Node interface
func ( d * Dir ) Sys ( ) interface { } {
2020-07-08 09:27:26 +00:00
return d . sys . Load ( )
2020-05-01 17:30:06 +00:00
}
// SetSys sets the underlying data source (can be nil) - satisfies Node interface
func ( d * Dir ) SetSys ( x interface { } ) {
2020-07-08 09:27:26 +00:00
d . sys . Store ( x )
2017-10-25 09:00:26 +00:00
}
2017-05-02 21:35:07 +00:00
// Inode returns the inode number - satisfies Node interface
func ( d * Dir ) Inode ( ) uint64 {
return d . inode
}
Spelling fixes
Fix spelling of: above, already, anonymous, associated,
authentication, bandwidth, because, between, blocks, calculate,
candidates, cautious, changelog, cleaner, clipboard, command,
completely, concurrently, considered, constructs, corrupt, current,
daemon, dependencies, deprecated, directory, dispatcher, download,
eligible, ellipsis, encrypter, endpoint, entrieslist, essentially,
existing writers, existing, expires, filesystem, flushing, frequently,
hierarchy, however, implementation, implements, inaccurate,
individually, insensitive, longer, maximum, metadata, modified,
multipart, namedirfirst, nextcloud, obscured, opened, optional,
owncloud, pacific, passphrase, password, permanently, persimmon,
positive, potato, protocol, quota, receiving, recommends, referring,
requires, revisited, satisfied, satisfies, satisfy, semver,
serialized, session, storage, strategies, stringlist, successful,
supported, surprise, temporarily, temporary, transactions, unneeded,
update, uploads, wrapped
Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 00:17:24 +00:00
// Node returns the Node associated with this - satisfies Noder interface
2017-05-02 21:35:07 +00:00
func ( d * Dir ) Node ( ) Node {
return d
}
2022-11-14 07:37:12 +00:00
// hasVirtual returns whether the directory has virtual entries
func ( d * Dir ) hasVirtual ( ) bool {
return d . _hasVirtual . Load ( )
}
// setHasVirtual sets the hasVirtual flag for the directory
func ( d * Dir ) setHasVirtual ( hasVirtual bool ) {
d . _hasVirtual . Store ( hasVirtual )
}
2020-06-23 12:04:27 +00:00
// ForgetAll forgets directory entries for this directory and any children.
2019-10-08 06:45:02 +00:00
//
// It does not invalidate or clear the cache of the parent directory.
2020-06-23 12:04:27 +00:00
//
// It returns true if the directory or any of its children had virtual entries
// so could not be forgotten. Children which didn't have virtual entries and
// children with virtual entries will be forgotten even if true is returned.
func ( d * Dir ) ForgetAll ( ) ( hasVirtual bool ) {
2022-11-10 11:34:57 +00:00
d . mu . RLock ( )
2020-06-23 12:04:27 +00:00
fs . Debugf ( d . path , "forgetting directory cache" )
for _ , node := range d . items {
if dir , ok := node . ( * Dir ) ; ok {
if dir . ForgetAll ( ) {
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( true )
2020-06-23 09:59:10 +00:00
}
2020-06-23 12:04:27 +00:00
}
2019-10-08 06:45:02 +00:00
}
2022-11-10 11:34:57 +00:00
d . mu . RUnlock ( )
d . mu . Lock ( )
defer d . mu . Unlock ( )
2022-08-14 02:56:32 +00:00
// Purge any unnecessary virtual entries
2020-11-10 12:58:03 +00:00
d . _purgeVirtual ( )
2020-06-23 12:04:27 +00:00
d . read = time . Time { }
2022-11-10 11:34:57 +00:00
2020-06-23 12:04:27 +00:00
// Check if this dir has virtual entries
if len ( d . virtual ) != 0 {
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( true )
2020-06-23 12:04:27 +00:00
}
2022-11-10 11:34:57 +00:00
2020-06-23 12:04:27 +00:00
// Don't clear directory entries if there are virtual entries in this
// directory or any children
2022-11-14 07:37:12 +00:00
if ! d . hasVirtual ( ) {
2020-06-23 12:04:27 +00:00
d . items = make ( map [ string ] Node )
2022-11-10 11:34:57 +00:00
d . cleanupTimer . Stop ( )
2020-06-23 12:04:27 +00:00
}
2022-11-10 11:34:57 +00:00
2022-11-14 07:37:12 +00:00
return d . hasVirtual ( )
2019-10-08 06:45:02 +00:00
}
2020-06-23 12:04:27 +00:00
// forgetDirPath clears the cache for itself and all subdirectories if
// they match the given path. The path is specified relative from the
// directory it is called from.
2019-10-08 06:45:02 +00:00
//
// It does not invalidate or clear the cache of the parent directory.
2020-06-23 12:04:27 +00:00
func ( d * Dir ) forgetDirPath ( relativePath string ) {
dir := d . cachedDir ( relativePath )
if dir == nil {
return
}
dir . ForgetAll ( )
2017-05-07 12:04:20 +00:00
}
2020-06-23 09:59:10 +00:00
// invalidateDir invalidates the directory cache for absPath relative to the root
2019-10-17 13:41:55 +00:00
func ( d * Dir ) invalidateDir ( absPath string ) {
node := d . vfs . root . cachedNode ( absPath )
if dir , ok := node . ( * Dir ) ; ok {
dir . mu . Lock ( )
if ! dir . read . IsZero ( ) {
fs . Debugf ( dir . path , "invalidating directory cache" )
dir . read = time . Time { }
}
dir . mu . Unlock ( )
}
}
// changeNotify invalidates the directory cache for the relativePath
// passed in.
//
// if entryType is a directory it invalidates the parent of the directory too.
func ( d * Dir ) changeNotify ( relativePath string , entryType fs . EntryType ) {
defer log . Trace ( d . path , "relativePath=%q, type=%v" , relativePath , entryType ) ( "" )
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
2019-10-17 13:41:55 +00:00
absPath := path . Join ( d . path , relativePath )
2019-11-06 21:48:43 +00:00
d . mu . RUnlock ( )
2020-02-28 14:44:15 +00:00
d . invalidateDir ( vfscommon . FindParent ( absPath ) )
2019-10-17 13:41:55 +00:00
if entryType == fs . EntryDirectory {
d . invalidateDir ( absPath )
}
}
2017-05-07 12:04:20 +00:00
// ForgetPath clears the cache for itself and all subdirectories if
// they match the given path. The path is specified relative from the
2018-09-25 16:27:37 +00:00
// directory it is called from. The cache of the parent directory is
// marked as stale, but not cleared otherwise.
2017-05-07 12:04:20 +00:00
// It is not possible to traverse the directory tree upwards, i.e.
// you cannot clear the cache for the Dir's ancestors or siblings.
2018-03-08 20:03:34 +00:00
func ( d * Dir ) ForgetPath ( relativePath string , entryType fs . EntryType ) {
2019-10-17 13:41:55 +00:00
defer log . Trace ( d . path , "relativePath=%q, type=%v" , relativePath , entryType ) ( "" )
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
absPath := path . Join ( d . path , relativePath )
d . mu . RUnlock ( )
if absPath != "" {
2020-02-28 14:44:15 +00:00
d . invalidateDir ( vfscommon . FindParent ( absPath ) )
2017-05-07 12:04:20 +00:00
}
2018-09-25 16:27:37 +00:00
if entryType == fs . EntryDirectory {
2019-10-08 06:45:02 +00:00
d . forgetDirPath ( relativePath )
2018-09-25 16:27:37 +00:00
}
2017-05-07 12:04:20 +00:00
}
2018-09-25 16:27:37 +00:00
// walk runs a function on all cached directories. It will be called
// on a directory's children first.
2019-11-06 21:48:43 +00:00
//
// The mutex will be held for the directory when fun is called
2018-09-25 16:27:37 +00:00
func ( d * Dir ) walk ( fun func ( * Dir ) ) {
2017-05-19 14:45:34 +00:00
d . mu . Lock ( )
defer d . mu . Unlock ( )
2018-03-01 15:51:05 +00:00
for _ , node := range d . items {
if dir , ok := node . ( * Dir ) ; ok {
2018-09-25 16:27:37 +00:00
dir . walk ( fun )
2017-05-07 12:04:20 +00:00
}
}
2018-09-25 16:27:37 +00:00
fun ( d )
}
2020-04-14 17:14:24 +00:00
// countActiveWriters returns the number of writers active in this
// directory and any subdirectories.
func ( d * Dir ) countActiveWriters ( ) ( writers int ) {
d . walk ( func ( d * Dir ) {
// NB d.mu is held by walk() here
fs . Debugf ( d . path , "Looking for writers" )
for leaf , item := range d . items {
fs . Debugf ( leaf , "reading active writers" )
if file , ok := item . ( * File ) ; ok {
n := file . activeWriters ( )
if n != 0 {
fs . Debugf ( file , "active writers %d" , n )
}
writers += n
}
}
} )
return writers
}
2018-09-25 16:27:37 +00:00
// age returns the duration since the last time the directory contents
Spelling fixes
Fix spelling of: above, already, anonymous, associated,
authentication, bandwidth, because, between, blocks, calculate,
candidates, cautious, changelog, cleaner, clipboard, command,
completely, concurrently, considered, constructs, corrupt, current,
daemon, dependencies, deprecated, directory, dispatcher, download,
eligible, ellipsis, encrypter, endpoint, entrieslist, essentially,
existing writers, existing, expires, filesystem, flushing, frequently,
hierarchy, however, implementation, implements, inaccurate,
individually, insensitive, longer, maximum, metadata, modified,
multipart, namedirfirst, nextcloud, obscured, opened, optional,
owncloud, pacific, passphrase, password, permanently, persimmon,
positive, potato, protocol, quota, receiving, recommends, referring,
requires, revisited, satisfied, satisfies, satisfy, semver,
serialized, session, storage, strategies, stringlist, successful,
supported, surprise, temporarily, temporary, transactions, unneeded,
update, uploads, wrapped
Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 00:17:24 +00:00
// was read and the content is considered stale. age will be 0 and
2018-09-25 16:27:37 +00:00
// stale true if the last read time is empty.
// age must be called with d.mu held.
2019-11-06 21:48:43 +00:00
func ( d * Dir ) _age ( when time . Time ) ( age time . Duration , stale bool ) {
2018-09-25 16:27:37 +00:00
if d . read . IsZero ( ) {
return age , true
2017-05-07 12:04:20 +00:00
}
2018-09-25 16:27:37 +00:00
age = when . Sub ( d . read )
stale = age > d . vfs . Opt . DirCacheTime
return
2017-05-07 12:04:20 +00:00
}
2021-03-17 15:01:35 +00:00
// renameTree renames the directories under this directory
//
// path should be the desired path
func ( d * Dir ) renameTree ( dirPath string ) {
d . mu . Lock ( )
defer d . mu . Unlock ( )
// Make sure the path is correct for each node
if d . path != dirPath {
fs . Debugf ( d . path , "Renaming to %q" , dirPath )
2024-04-28 11:47:24 +00:00
delete ( d . parent . items , name ( d . path ) )
2021-03-17 15:01:35 +00:00
d . path = dirPath
2024-04-28 11:47:24 +00:00
d . parent . items [ name ( d . path ) ] = d
2021-03-17 15:01:35 +00:00
d . entry = fs . NewDirCopy ( context . TODO ( ) , d . entry ) . SetRemote ( dirPath )
}
2023-03-06 11:49:03 +00:00
// Do the same to any child directories and files
2021-03-17 15:01:35 +00:00
for leaf , node := range d . items {
2023-03-06 11:49:03 +00:00
switch x := node . ( type ) {
case * Dir :
x . renameTree ( path . Join ( dirPath , leaf ) )
case * File :
x . renameDir ( dirPath )
default :
panic ( "bad dir entry" )
2021-03-17 15:01:35 +00:00
}
}
}
2017-05-02 21:35:07 +00:00
// rename should be called after the directory is renamed
//
// Reset the directory to new state, discarding all the objects and
// reading everything again
2017-06-30 12:37:29 +00:00
func ( d * Dir ) rename ( newParent * Dir , fsDir fs . Directory ) {
2017-05-07 12:04:20 +00:00
d . ForgetAll ( )
2021-03-17 15:01:35 +00:00
2020-07-08 09:27:26 +00:00
d . modTimeMu . Lock ( )
d . modTime = fsDir . ModTime ( context . TODO ( ) )
d . modTimeMu . Unlock ( )
2019-11-06 21:48:43 +00:00
d . mu . Lock ( )
2021-03-17 09:48:25 +00:00
oldPath := d . path
2017-10-26 15:05:34 +00:00
d . parent = newParent
d . entry = fsDir
2017-06-30 12:37:29 +00:00
d . path = fsDir . Remote ( )
2021-03-17 09:48:25 +00:00
newPath := d . path
2024-04-28 11:47:24 +00:00
delete ( d . parent . items , name ( oldPath ) )
d . parent . items [ name ( d . path ) ] = d
2017-05-02 21:35:07 +00:00
d . read = time . Time { }
2019-11-06 21:48:43 +00:00
d . mu . Unlock ( )
2021-03-17 09:48:25 +00:00
2021-03-17 15:01:35 +00:00
// Rename any remaining items in the tree that we couldn't forget
d . renameTree ( d . path )
2021-03-17 09:48:25 +00:00
// Rename in the cache
if d . vfs . cache != nil && d . vfs . cache . DirExists ( oldPath ) {
if err := d . vfs . cache . DirRename ( oldPath , newPath ) ; err != nil {
fs . Infof ( d , "Dir.Rename failed in Cache: %v" , err )
}
}
2017-05-02 21:35:07 +00:00
}
2024-04-28 11:47:24 +00:00
// convert path to name
func name ( p string ) string {
p = path . Base ( p )
if p == "." {
p = "/"
}
return p
}
2017-05-02 21:35:07 +00:00
// addObject adds a new object or directory to the directory
//
2020-06-23 09:59:10 +00:00
// The name passed in is marked as virtual as it hasn't been read from a remote
// directory listing.
//
2017-05-02 21:35:07 +00:00
// note that we add new objects rather than updating old ones
2017-10-26 16:21:03 +00:00
func ( d * Dir ) addObject ( node Node ) {
2017-05-02 21:35:07 +00:00
d . mu . Lock ( )
2020-06-23 09:59:10 +00:00
leaf := node . Name ( )
d . items [ leaf ] = node
if d . virtual == nil {
d . virtual = make ( map [ string ] vState )
}
2020-11-10 12:58:03 +00:00
vAdd := vAddFile
if node . IsDir ( ) {
vAdd = vAddDir
}
2020-06-23 09:59:10 +00:00
d . virtual [ leaf ] = vAdd
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( true )
2020-06-23 09:59:10 +00:00
fs . Debugf ( d . path , "Added virtual directory entry %v: %q" , vAdd , leaf )
2017-05-02 21:35:07 +00:00
d . mu . Unlock ( )
}
2020-06-23 09:59:10 +00:00
// AddVirtual adds a virtual object of name and size to the directory
//
// This will be replaced with a real object when it is read back from the
// remote.
//
// This is used to add directory entries while things are uploading
func ( d * Dir ) AddVirtual ( leaf string , size int64 , isDir bool ) {
var node Node
d . mu . RLock ( )
dPath := d . path
_ , found := d . items [ leaf ]
d . mu . RUnlock ( )
if found {
// Don't overwrite existing objects
return
}
if isDir {
remote := path . Join ( dPath , leaf )
entry := fs . NewDir ( remote , time . Now ( ) )
node = newDir ( d . vfs , d . f , d , entry )
} else {
f := newFile ( d , dPath , nil , leaf )
f . setSize ( size )
node = f
}
d . addObject ( node )
}
2017-05-02 21:35:07 +00:00
// delObject removes an object from the directory
2020-06-23 09:59:10 +00:00
//
// The name passed in is marked as virtual as the delete it hasn't been read
// from a remote directory listing.
2017-05-02 21:35:07 +00:00
func ( d * Dir ) delObject ( leaf string ) {
d . mu . Lock ( )
2018-03-01 15:51:05 +00:00
delete ( d . items , leaf )
2020-06-23 09:59:10 +00:00
if d . virtual == nil {
d . virtual = make ( map [ string ] vState )
}
d . virtual [ leaf ] = vDel
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( true )
2020-06-23 09:59:10 +00:00
fs . Debugf ( d . path , "Added virtual directory entry %v: %q" , vDel , leaf )
2017-05-02 21:35:07 +00:00
d . mu . Unlock ( )
}
2020-06-23 09:59:10 +00:00
// DelVirtual removes an object from the directory listing
//
// It marks it as removed until it has confirmed the object is missing when the
2020-11-10 12:58:03 +00:00
// directory entries are re-read in.
2020-06-23 09:59:10 +00:00
//
// This is used to remove directory entries after things have been deleted or
// renamed but before we've had confirmation from the backend.
func ( d * Dir ) DelVirtual ( leaf string ) {
d . delObject ( leaf )
}
2017-07-22 09:55:41 +00:00
// read the directory and sets d.items - must be called with the lock held
func ( d * Dir ) _readDir ( ) error {
2017-05-02 21:35:07 +00:00
when := time . Now ( )
2019-11-06 21:48:43 +00:00
if age , stale := d . _age ( when ) ; stale {
2018-09-25 16:27:37 +00:00
if age != 0 {
fs . Debugf ( d . path , "Re-reading directory (%v old)" , age )
2017-05-02 21:35:07 +00:00
}
2018-09-25 16:27:37 +00:00
} else {
return nil
2017-05-02 21:35:07 +00:00
}
2019-06-17 08:34:30 +00:00
entries , err := list . DirSorted ( context . TODO ( ) , d . f , false , d . path )
2017-05-02 21:35:07 +00:00
if err == fs . ErrorDirNotFound {
// We treat directory not found as empty because we
// create directories on the fly
} else if err != nil {
return err
}
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
if d . vfs . Opt . BlockNormDupes { // do this only if requested, as it will have a performance hit
ci := fs . GetConfig ( context . TODO ( ) )
// sort entries such that NFD comes before NFC of same name
sort . Slice ( entries , func ( i , j int ) bool {
if entries [ i ] != entries [ j ] && fs . DirEntryType ( entries [ i ] ) == fs . DirEntryType ( entries [ j ] ) && norm . NFC . String ( entries [ i ] . Remote ( ) ) == norm . NFC . String ( entries [ j ] . Remote ( ) ) {
if norm . NFD . IsNormalString ( entries [ i ] . Remote ( ) ) && ! norm . NFD . IsNormalString ( entries [ j ] . Remote ( ) ) {
return true
}
}
return entries . Less ( i , j )
} )
// detect dupes, remove them from the list and log an error
normalizedNames := make ( map [ string ] struct { } , entries . Len ( ) )
filteredEntries := make ( fs . DirEntries , 0 )
for _ , e := range entries {
normName := fmt . Sprintf ( "%s-%T" , operations . ToNormal ( e . Remote ( ) , ! ci . NoUnicodeNormalization , ( ci . IgnoreCaseSync || d . vfs . Opt . CaseInsensitive ) ) , e ) // include type to track objects and dirs separately
_ , found := normalizedNames [ normName ]
if found {
fs . Errorf ( e . Remote ( ) , "duplicate normalized names detected - skipping" )
continue
}
normalizedNames [ normName ] = struct { } { }
filteredEntries = append ( filteredEntries , e )
}
entries = filteredEntries
}
2018-07-27 14:36:22 +00:00
err = d . _readDirFromEntries ( entries , nil , time . Time { } )
if err != nil {
return err
}
d . read = when
2022-11-10 11:34:57 +00:00
d . cleanupTimer . Reset ( d . vfs . Opt . DirCacheTime * 2 )
2018-07-27 14:36:22 +00:00
return nil
}
2018-07-31 10:34:59 +00:00
// update d.items for each dir in the DirTree below this one and
// set the last read time - must be called with the lock held
2019-06-27 14:54:43 +00:00
func ( d * Dir ) _readDirFromDirTree ( dirTree dirtree . DirTree , when time . Time ) error {
2018-07-27 14:36:22 +00:00
return d . _readDirFromEntries ( dirTree [ d . path ] , dirTree , when )
}
2020-11-10 12:58:03 +00:00
// Remove the virtual directory entry leaf
func ( d * Dir ) _deleteVirtual ( name string ) {
virtualState , ok := d . virtual [ name ]
if ! ok {
return
}
delete ( d . virtual , name )
if len ( d . virtual ) == 0 {
d . virtual = nil
2022-11-14 07:37:12 +00:00
d . setHasVirtual ( false )
2020-11-10 12:58:03 +00:00
}
fs . Debugf ( d . path , "Removed virtual directory entry %v: %q" , virtualState , name )
}
// Purge virtual entries assuming the directory has just been re-read
//
// Remove all the entries except:
//
// 1) vDirAdd on remotes which can't have empty directories. These will remain
// virtual as long as the directory is empty. When the directory becomes real
// (ie files are added) the virtual directory will be removed. This means that
// directories will disappear when the last file is deleted which is probably
// OK.
//
// 2) vFileAdd that are being written or uploaded
func ( d * Dir ) _purgeVirtual ( ) {
canHaveEmptyDirectories := d . f . Features ( ) . CanHaveEmptyDirectories
for name , virtualState := range d . virtual {
switch virtualState {
case vAddDir :
if canHaveEmptyDirectories {
// if remote can have empty directories then a
// new dir will be read in the listing
d . _deleteVirtual ( name )
2022-06-08 20:25:17 +00:00
//} else {
2020-11-10 12:58:03 +00:00
// leave the empty directory marker
}
case vAddFile :
// Delete all virtual file adds that have finished uploading
node , ok := d . items [ name ]
if ! ok {
// if the object has disappeared somehow then remove the virtual
d . _deleteVirtual ( name )
continue
}
f , ok := node . ( * File )
if ! ok {
// if the object isn't a file then remove the virtual as it is wrong
d . _deleteVirtual ( name )
continue
}
if f . writingInProgress ( ) {
// if writing in progress then leave virtual
continue
}
if d . vfs . Opt . CacheMode >= vfscommon . CacheModeMinimal && d . vfs . cache . InUse ( f . Path ( ) ) {
// if object in use or dirty then leave virtual
continue
}
d . _deleteVirtual ( name )
default :
d . _deleteVirtual ( name )
}
}
}
// Manage the virtuals in a listing
//
// This keeps a record of the names listed in this directory so far
type manageVirtuals map [ string ] struct { }
// Create a new manageVirtuals and purge the d.virtuals of any entries which can
// be removed.
//
// must be called with the Dir lock held
func ( d * Dir ) _newManageVirtuals ( ) manageVirtuals {
tv := make ( manageVirtuals )
d . _purgeVirtual ( )
return tv
}
// This should be called for every entry added to the directory
//
2022-08-05 15:35:41 +00:00
// It returns true if this entry should be skipped.
2020-11-10 12:58:03 +00:00
//
// must be called with the Dir lock held
func ( mv manageVirtuals ) add ( d * Dir , name string ) bool {
// Keep a record of all names listed
mv [ name ] = struct { } { }
// Remove virtuals if possible
switch d . virtual [ name ] {
case vAddFile , vAddDir :
// item was added to the dir but since it is found in a
// listing is no longer virtual
d . _deleteVirtual ( name )
case vDel :
// item is deleted from the dir so skip it
return true
case vOK :
}
return false
}
// This should be called after the directory entry is read to update d.items
// with virtual entries
//
// must be called with the Dir lock held
func ( mv manageVirtuals ) end ( d * Dir ) {
// delete unused d.items
for name := range d . items {
if _ , ok := mv [ name ] ; ! ok {
// name was previously in the directory but wasn't found
// in the current listing
switch d . virtual [ name ] {
case vAddFile , vAddDir :
// virtually added so leave virtual item
default :
// otherwise delete it
delete ( d . items , name )
}
}
}
// delete unused d.virtual~s
for name , virtualState := range d . virtual {
if _ , ok := mv [ name ] ; ! ok {
// name exists as a virtual but isn't in the current
// listing so if it is a virtual delete we can remove it
// as it is no longer needed.
if virtualState == vDel {
d . _deleteVirtual ( name )
}
}
}
}
2018-07-31 10:34:59 +00:00
// update d.items and if dirTree is not nil update each dir in the DirTree below this one and
// set the last read time - must be called with the lock held
2019-06-27 14:54:43 +00:00
func ( d * Dir ) _readDirFromEntries ( entries fs . DirEntries , dirTree dirtree . DirTree , when time . Time ) error {
2018-07-27 14:36:22 +00:00
var err error
2020-11-10 12:58:03 +00:00
mv := d . _newManageVirtuals ( )
2017-05-02 21:35:07 +00:00
for _ , entry := range entries {
2018-03-01 15:51:05 +00:00
name := path . Base ( entry . Remote ( ) )
2018-04-24 21:28:37 +00:00
if name == "." || name == ".." {
continue
}
2018-03-01 15:51:05 +00:00
node := d . items [ name ]
2020-11-10 12:58:03 +00:00
if mv . add ( d , name ) {
2020-06-23 09:59:10 +00:00
continue
}
2017-05-02 21:35:07 +00:00
switch item := entry . ( type ) {
case fs . Object :
obj := item
2018-03-01 15:51:05 +00:00
// Reuse old file value if it exists
if file , ok := node . ( * File ) ; node != nil && ok {
file . setObjectNoUpdate ( obj )
} else {
2020-04-14 16:55:18 +00:00
node = newFile ( d , d . path , obj , name )
2018-03-01 15:51:05 +00:00
}
2017-06-30 12:37:29 +00:00
case fs . Directory :
2018-03-01 15:51:05 +00:00
// Reuse old dir value if it exists
if node == nil || ! node . IsDir ( ) {
2018-07-27 14:36:22 +00:00
node = newDir ( d . vfs , d . f , d , item )
}
2023-09-24 21:51:08 +00:00
dir := node . ( * Dir )
dir . mu . Lock ( )
dir . modTime = item . ModTime ( context . TODO ( ) )
2018-07-27 14:36:22 +00:00
if dirTree != nil {
err = dir . _readDirFromDirTree ( dirTree , when )
if err != nil {
dir . read = time . Time { }
} else {
dir . read = when
2022-11-10 11:34:57 +00:00
dir . cleanupTimer . Reset ( d . vfs . Opt . DirCacheTime * 2 )
2018-07-27 14:36:22 +00:00
}
2023-09-24 21:51:08 +00:00
}
dir . mu . Unlock ( )
if err != nil {
return err
2017-05-02 21:35:07 +00:00
}
default :
2021-11-04 10:12:57 +00:00
err = fmt . Errorf ( "unknown type %T" , item )
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "readDir error: %v" , err )
2017-05-02 21:35:07 +00:00
return err
}
2018-03-01 15:51:05 +00:00
d . items [ name ] = node
}
2020-11-10 12:58:03 +00:00
mv . end ( d )
2018-07-27 14:36:22 +00:00
return nil
}
2018-07-29 22:30:32 +00:00
// readDirTree forces a refresh of the complete directory tree
2018-07-27 14:36:22 +00:00
func ( d * Dir ) readDirTree ( ) error {
2019-11-06 21:48:43 +00:00
d . mu . RLock ( )
2019-03-04 17:25:04 +00:00
f , path := d . f , d . path
2019-11-06 21:48:43 +00:00
d . mu . RUnlock ( )
2018-07-27 14:36:22 +00:00
when := time . Now ( )
2019-03-04 17:25:04 +00:00
fs . Debugf ( path , "Reading directory tree" )
2019-06-17 08:34:30 +00:00
dt , err := walk . NewDirTree ( context . TODO ( ) , f , path , false , - 1 )
2018-07-27 14:36:22 +00:00
if err != nil {
return err
}
2019-03-04 17:25:04 +00:00
d . mu . Lock ( )
defer d . mu . Unlock ( )
d . read = time . Time { }
2018-07-27 14:36:22 +00:00
err = d . _readDirFromDirTree ( dt , when )
if err != nil {
return err
}
fs . Debugf ( d . path , "Reading directory tree done in %s" , time . Since ( when ) )
2017-05-02 21:35:07 +00:00
d . read = when
2022-11-10 11:34:57 +00:00
d . cleanupTimer . Reset ( d . vfs . Opt . DirCacheTime * 2 )
2017-05-02 21:35:07 +00:00
return nil
}
2018-07-29 22:30:32 +00:00
// readDir forces a refresh of the directory
func ( d * Dir ) readDir ( ) error {
d . mu . Lock ( )
defer d . mu . Unlock ( )
d . read = time . Time { }
return d . _readDir ( )
}
2017-10-29 11:36:38 +00:00
// stat a single item in the directory
2017-05-02 21:35:07 +00:00
//
// returns ENOENT if not found.
2019-09-04 20:30:48 +00:00
// returns a custom error if directory on a case-insensitive file system
// contains files with names that differ only by case.
2017-10-29 11:36:38 +00:00
func ( d * Dir ) stat ( leaf string ) ( Node , error ) {
2017-07-22 09:55:41 +00:00
d . mu . Lock ( )
defer d . mu . Unlock ( )
err := d . _readDir ( )
2017-05-02 21:35:07 +00:00
if err != nil {
return nil , err
}
item , ok := d . items [ leaf ]
2019-09-04 20:30:48 +00:00
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
ci := fs . GetConfig ( context . TODO ( ) )
normUnicode := ! ci . NoUnicodeNormalization
normCase := ci . IgnoreCaseSync || d . vfs . Opt . CaseInsensitive
if ! ok && ( normUnicode || normCase ) {
leafNormalized := operations . ToNormal ( leaf , normUnicode , normCase ) // this handles both case and unicode normalization
2019-09-04 20:30:48 +00:00
for name , node := range d . items {
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
if operations . ToNormal ( name , normUnicode , normCase ) == leafNormalized {
2019-09-04 20:30:48 +00:00
if ok {
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
// duplicate normalized match is an error
return nil , fmt . Errorf ( "duplicate filename %q detected with case/unicode normalization settings" , leaf )
2019-09-04 20:30:48 +00:00
}
vfs: fix unicode normalization on macOS - fixes #7072
Before this change, the VFS layer did not properly handle unicode normalization,
which caused problems particularly for users of macOS. While attempts were made
to handle it with various `-o modules=iconv` combinations, this was an imperfect
solution, as no one combination allowed both NFC and NFD content to
simultaneously be both visible and editable via Finder.
After this change, the VFS supports `--no-unicode-normalization` (default `false`)
via the existing `--vfs-case-insensitive` logic, which is extended to apply to both
case insensitivity and unicode normalization form.
This change also adds an additional flag, `--vfs-block-norm-dupes`, to address a
probably rare but potentially possible scenario where a directory contains
multiple duplicate filenames after applying case and unicode normalization
settings. In such a scenario, this flag (disabled by default) hides the
duplicates. This comes with a performance tradeoff, as rclone will have to scan
the entire directory for duplicates when listing a directory. For this reason,
it is recommended to leave this disabled if not needed. However, macOS users may
wish to consider using it, as otherwise, if a remote directory contains both NFC
and NFD versions of the same filename, an odd situation will occur: both
versions of the file will be visible in the mount, and both will appear to be
editable, however, editing either version will actually result in only the NFD
version getting edited under the hood. `--vfs-block-norm-dupes` prevents this
confusion by detecting this scenario, hiding the duplicates, and logging an
error, similar to how this is handled in `rclone sync`.
2024-02-05 07:58:11 +00:00
// found a normalized match
2019-09-04 20:30:48 +00:00
ok = true
item = node
}
}
}
2017-05-02 21:35:07 +00:00
if ! ok {
return nil , ENOENT
}
return item , nil
}
// Check to see if a directory is empty
func ( d * Dir ) isEmpty ( ) ( bool , error ) {
2017-07-22 09:55:41 +00:00
d . mu . Lock ( )
defer d . mu . Unlock ( )
err := d . _readDir ( )
2017-05-02 21:35:07 +00:00
if err != nil {
return false , err
}
return len ( d . items ) == 0 , nil
}
// ModTime returns the modification time of the directory
func ( d * Dir ) ModTime ( ) time . Time {
2020-07-08 09:27:26 +00:00
d . modTimeMu . Lock ( )
defer d . modTimeMu . Unlock ( )
2017-05-09 10:39:33 +00:00
// fs.Debugf(d.path, "Dir.ModTime %v", d.modTime)
2017-05-02 21:35:07 +00:00
return d . modTime
}
2017-10-25 09:00:26 +00:00
// Size of the directory
func ( d * Dir ) Size ( ) int64 {
return 0
}
2017-05-02 21:35:07 +00:00
// SetModTime sets the modTime for this dir
func ( d * Dir ) SetModTime ( modTime time . Time ) error {
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-05-11 12:15:51 +00:00
return EROFS
}
2020-07-08 09:27:26 +00:00
d . modTimeMu . Lock ( )
2017-05-02 21:35:07 +00:00
d . modTime = modTime
2020-07-08 09:27:26 +00:00
d . modTimeMu . Unlock ( )
2017-05-02 21:35:07 +00:00
return nil
}
2018-09-25 16:27:37 +00:00
func ( d * Dir ) cachedDir ( relativePath string ) ( dir * Dir ) {
dir , _ = d . cachedNode ( relativePath ) . ( * Dir )
return
}
func ( d * Dir ) cachedNode ( relativePath string ) Node {
segments := strings . Split ( strings . Trim ( relativePath , "/" ) , "/" )
var node Node = d
for _ , s := range segments {
if s == "" {
continue
}
if dir , ok := node . ( * Dir ) ; ok {
dir . mu . Lock ( )
node = dir . items [ s ]
dir . mu . Unlock ( )
if node != nil {
continue
}
}
return nil
}
return node
}
2017-10-29 11:36:38 +00:00
// Stat looks up a specific entry in the receiver.
2017-05-02 21:35:07 +00:00
//
2017-10-29 11:36:38 +00:00
// Stat should return a Node corresponding to the entry. If the
// name does not exist in the directory, Stat should return ENOENT.
2017-05-02 21:35:07 +00:00
//
2017-10-29 11:36:38 +00:00
// Stat need not to handle the names "." and "..".
func ( d * Dir ) Stat ( name string ) ( node Node , err error ) {
// fs.Debugf(path, "Dir.Stat")
node , err = d . stat ( name )
2017-05-02 21:35:07 +00:00
if err != nil {
if err != ENOENT {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Stat error: %v" , err )
2017-05-02 21:35:07 +00:00
}
return nil , err
}
2017-10-29 11:36:38 +00:00
// fs.Debugf(path, "Dir.Stat OK")
2017-10-26 16:21:03 +00:00
return node , nil
2017-05-02 21:35:07 +00:00
}
2017-10-27 21:07:59 +00:00
// ReadDirAll reads the contents of the directory sorted
func ( d * Dir ) ReadDirAll ( ) ( items Nodes , err error ) {
2017-05-09 10:39:33 +00:00
// fs.Debugf(d.path, "Dir.ReadDirAll")
2017-07-22 09:55:41 +00:00
d . mu . Lock ( )
err = d . _readDir ( )
2017-05-02 21:35:07 +00:00
if err != nil {
fs . Debugf ( d . path , "Dir.ReadDirAll error: %v" , err )
2020-02-27 15:50:41 +00:00
d . mu . Unlock ( )
2017-05-02 21:35:07 +00:00
return nil , err
}
for _ , item := range d . items {
items = append ( items , item )
}
2020-02-27 15:50:41 +00:00
d . mu . Unlock ( )
2017-10-27 21:07:59 +00:00
sort . Sort ( items )
2017-05-09 10:39:33 +00:00
// fs.Debugf(d.path, "Dir.ReadDirAll OK with %d entries", len(items))
2017-05-02 21:35:07 +00:00
return items , nil
}
2017-11-06 21:38:52 +00:00
// accessModeMask masks off the read modes from the flags
const accessModeMask = ( os . O_RDONLY | os . O_WRONLY | os . O_RDWR )
2017-10-30 10:14:39 +00:00
// Open the directory according to the flags provided
func ( d * Dir ) Open ( flags int ) ( fd Handle , err error ) {
2017-11-06 21:38:52 +00:00
rdwrMode := flags & accessModeMask
2017-10-30 10:14:39 +00:00
if rdwrMode != os . O_RDONLY {
fs . Errorf ( d , "Can only open directories read only" )
2017-10-31 15:33:08 +00:00
return nil , EPERM
2017-10-30 10:14:39 +00:00
}
return newDirHandle ( d ) , nil
}
2017-11-06 12:22:45 +00:00
// Create makes a new file node
2017-12-07 12:34:18 +00:00
func ( d * Dir ) Create ( name string , flags int ) ( * File , error ) {
2017-11-06 12:22:45 +00:00
// fs.Debugf(path, "Dir.Create")
2021-03-31 13:48:02 +00:00
// Return existing node if one exists
node , err := d . stat ( name )
switch err {
case ENOENT :
// not found, carry on
case nil :
// found so check what it is
if node . IsFile ( ) {
return node . ( * File ) , err
}
return nil , EEXIST // EISDIR would be better but we don't have that
default :
// a different error - report
fs . Errorf ( d , "Dir.Create stat failed: %v" , err )
return nil , err
}
// node doesn't exist so create it
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-11-06 12:22:45 +00:00
return nil , EROFS
2017-05-11 12:15:51 +00:00
}
2023-05-15 19:52:03 +00:00
if err = d . SetModTime ( time . Now ( ) ) ; err != nil {
fs . Errorf ( d , "Dir.Create failed to set modtime on parent dir: %v" , err )
return nil , err
}
2017-11-18 11:47:21 +00:00
// This gets added to the directory when the file is opened for write
2020-04-14 16:55:18 +00:00
return newFile ( d , d . Path ( ) , nil , name ) , nil
2017-05-02 21:35:07 +00:00
}
// Mkdir creates a new directory
func ( d * Dir ) Mkdir ( name string ) ( * Dir , error ) {
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-05-11 12:15:51 +00:00
return nil , EROFS
}
2017-05-02 21:35:07 +00:00
path := path . Join ( d . path , name )
2019-02-27 16:38:32 +00:00
node , err := d . stat ( name )
switch err {
case ENOENT :
// not found, carry on
case nil :
// found so check what it is
if node . IsDir ( ) {
return node . ( * Dir ) , err
}
return nil , EEXIST
default :
// a different error - report
fs . Errorf ( d , "Dir.Mkdir failed to read directory: %v" , err )
return nil , err
}
2017-05-09 10:39:33 +00:00
// fs.Debugf(path, "Dir.Mkdir")
2019-06-17 08:34:30 +00:00
err = d . f . Mkdir ( context . TODO ( ) , path )
2017-05-02 21:35:07 +00:00
if err != nil {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Mkdir failed to create directory: %v" , err )
2017-05-02 21:35:07 +00:00
return nil , err
}
2017-06-30 12:37:29 +00:00
fsDir := fs . NewDir ( path , time . Now ( ) )
2017-10-28 19:01:34 +00:00
dir := newDir ( d . vfs , d . f , d , fsDir )
2017-10-26 16:21:03 +00:00
d . addObject ( dir )
2023-05-15 19:52:03 +00:00
if err = d . SetModTime ( time . Now ( ) ) ; err != nil {
fs . Errorf ( d , "Dir.Mkdir failed to set modtime on parent dir: %v" , err )
return nil , err
}
2017-05-09 10:39:33 +00:00
// fs.Debugf(path, "Dir.Mkdir OK")
2017-05-02 21:35:07 +00:00
return dir , nil
}
2017-10-26 15:55:40 +00:00
// Remove the directory
func ( d * Dir ) Remove ( ) error {
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-10-26 15:55:40 +00:00
return EROFS
}
// Check directory is empty first
empty , err := d . isEmpty ( )
if err != nil {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Remove dir error: %v" , err )
2017-10-26 15:55:40 +00:00
return err
}
if ! empty {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Remove not empty" )
2017-10-26 15:55:40 +00:00
return ENOTEMPTY
}
// remove directory
2019-06-17 08:34:30 +00:00
err = d . f . Rmdir ( context . TODO ( ) , d . path )
2017-10-26 15:55:40 +00:00
if err != nil {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Remove failed to remove directory: %v" , err )
2017-10-26 15:55:40 +00:00
return err
}
// Remove the item from the parent directory listing
if d . parent != nil {
d . parent . delObject ( d . Name ( ) )
}
return nil
}
// RemoveAll removes the directory and any contents recursively
func ( d * Dir ) RemoveAll ( ) error {
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-10-26 15:55:40 +00:00
return EROFS
}
// Remove contents of the directory
2017-10-26 16:21:03 +00:00
nodes , err := d . ReadDirAll ( )
2017-10-26 15:55:40 +00:00
if err != nil {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.RemoveAll failed to read directory: %v" , err )
2017-10-26 15:55:40 +00:00
return err
}
2017-10-26 16:21:03 +00:00
for _ , node := range nodes {
err = node . RemoveAll ( )
2017-10-26 15:55:40 +00:00
if err != nil {
2017-11-18 11:47:21 +00:00
fs . Errorf ( node . Path ( ) , "Dir.RemoveAll failed to remove: %v" , err )
2017-10-26 15:55:40 +00:00
return err
}
}
return d . Remove ( )
}
2017-10-26 16:02:48 +00:00
// DirEntry returns the underlying fs.DirEntry
func ( d * Dir ) DirEntry ( ) ( entry fs . DirEntry ) {
return d . entry
}
2017-10-26 14:37:45 +00:00
// RemoveName removes the entry with the given name from the receiver,
// which must be a directory. The entry to be removed may correspond
// to a file (unlink) or to a directory (rmdir).
func ( d * Dir ) RemoveName ( name string ) error {
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-05-11 12:15:51 +00:00
return EROFS
}
2017-05-09 10:39:33 +00:00
// fs.Debugf(path, "Dir.Remove")
2017-10-29 11:36:38 +00:00
node , err := d . stat ( name )
2017-05-02 21:35:07 +00:00
if err != nil {
2017-10-30 10:14:39 +00:00
fs . Errorf ( d , "Dir.Remove error: %v" , err )
2017-05-02 21:35:07 +00:00
return err
}
2023-05-15 19:52:03 +00:00
if err = d . SetModTime ( time . Now ( ) ) ; err != nil {
fs . Errorf ( d , "Dir.Remove failed to set modtime on parent dir: %v" , err )
return err
}
2017-10-26 16:21:03 +00:00
return node . Remove ( )
2017-05-02 21:35:07 +00:00
}
// Rename the file
func ( d * Dir ) Rename ( oldName , newName string , destDir * Dir ) error {
2021-03-17 15:01:35 +00:00
// fs.Debugf(d, "BEFORE\n%s", d.dump())
2017-10-29 11:00:56 +00:00
if d . vfs . Opt . ReadOnly {
2017-05-11 12:15:51 +00:00
return EROFS
}
2017-05-02 21:35:07 +00:00
oldPath := path . Join ( d . path , oldName )
newPath := path . Join ( destDir . path , newName )
2017-05-09 10:39:33 +00:00
// fs.Debugf(oldPath, "Dir.Rename to %q", newPath)
2017-10-29 11:36:38 +00:00
oldNode , err := d . stat ( oldName )
2017-05-02 21:35:07 +00:00
if err != nil {
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
2017-10-26 16:21:03 +00:00
switch x := oldNode . DirEntry ( ) . ( type ) {
2017-11-18 11:47:21 +00:00
case nil :
2018-05-24 18:45:11 +00:00
if oldFile , ok := oldNode . ( * File ) ; ok {
2019-06-17 08:34:30 +00:00
if err = oldFile . rename ( context . TODO ( ) , destDir , newName ) ; err != nil {
2018-05-24 18:45:11 +00:00
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
} else {
fs . Errorf ( oldPath , "Dir.Rename can't rename open file that is not a vfs.File" )
return EPERM
2017-05-02 21:35:07 +00:00
}
2018-05-24 18:45:11 +00:00
case fs . Object :
if oldFile , ok := oldNode . ( * File ) ; ok {
2019-06-17 08:34:30 +00:00
if err = oldFile . rename ( context . TODO ( ) , destDir , newName ) ; err != nil {
2018-05-24 18:45:11 +00:00
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
} else {
2021-11-04 10:12:57 +00:00
err := fmt . Errorf ( "Fs %q can't rename file that is not a vfs.File" , d . f )
2017-05-02 21:35:07 +00:00
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
2017-06-30 12:37:29 +00:00
case fs . Directory :
2019-01-15 11:45:44 +00:00
features := d . f . Features ( )
if features . DirMove == nil && features . Move == nil && features . Copy == nil {
2021-11-04 10:12:57 +00:00
err := fmt . Errorf ( "Fs %q can't rename directories (no DirMove, Move or Copy)" , d . f )
2017-05-02 21:35:07 +00:00
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
2017-06-30 12:37:29 +00:00
srcRemote := x . Remote ( )
2017-05-02 21:35:07 +00:00
dstRemote := newPath
2019-06-17 08:34:30 +00:00
err = operations . DirMove ( context . TODO ( ) , d . f , srcRemote , dstRemote )
2017-05-02 21:35:07 +00:00
if err != nil {
fs . Errorf ( oldPath , "Dir.Rename error: %v" , err )
return err
}
2019-06-17 08:34:30 +00:00
newDir := fs . NewDirCopy ( context . TODO ( ) , x ) . SetRemote ( newPath )
2017-05-02 21:35:07 +00:00
// Update the node with the new details
if oldNode != nil {
if oldDir , ok := oldNode . ( * Dir ) ; ok {
2017-11-18 11:47:21 +00:00
fs . Debugf ( x , "Updating dir with %v %p" , newDir , oldDir )
2017-05-02 21:35:07 +00:00
oldDir . rename ( destDir , newDir )
}
}
default :
2021-11-04 10:12:57 +00:00
err = fmt . Errorf ( "unknown type %T" , oldNode )
2019-02-27 16:38:32 +00:00
fs . Errorf ( d . path , "Dir.Rename error: %v" , err )
2017-05-02 21:35:07 +00:00
return err
}
// Show moved - delete from old dir and add to new
d . delObject ( oldName )
2017-10-26 16:21:03 +00:00
destDir . addObject ( oldNode )
2023-05-15 19:52:03 +00:00
if err = d . SetModTime ( time . Now ( ) ) ; err != nil {
fs . Errorf ( d , "Dir.Rename failed to set modtime on parent dir: %v" , err )
return err
}
2017-05-02 21:35:07 +00:00
2017-05-09 10:39:33 +00:00
// fs.Debugf(newPath, "Dir.Rename renamed from %q", oldPath)
2021-03-17 15:01:35 +00:00
// fs.Debugf(d, "AFTER\n%s", d.dump())
2017-05-02 21:35:07 +00:00
return nil
}
2017-11-18 15:48:49 +00:00
// Sync the directory
2017-05-02 21:35:07 +00:00
//
// Note that we don't do anything except return OK
2017-11-18 15:48:49 +00:00
func ( d * Dir ) Sync ( ) error {
2017-05-02 21:35:07 +00:00
return nil
}
2017-10-29 11:00:56 +00:00
// VFS returns the instance of the VFS
func ( d * Dir ) VFS ( ) * VFS {
2020-04-14 17:14:24 +00:00
// No locking required
2017-10-29 11:00:56 +00:00
return d . vfs
}
2017-11-06 21:38:52 +00:00
2020-04-14 17:14:24 +00:00
// Fs returns the Fs that the Dir is on
func ( d * Dir ) Fs ( ) fs . Fs {
// No locking required
return d . f
}
2017-11-06 21:38:52 +00:00
// Truncate changes the size of the named file.
func ( d * Dir ) Truncate ( size int64 ) error {
return ENOSYS
}