76f5e273d2
Before this change, change notify polls would clear the directory cache recursively. So uploading a file to the root would clear the entire directory cache. After this change we just invalidate the directory cache of the parent directory of the item and if the item was a directory we invalidate it too.
511 lines
14 KiB
Go
511 lines
14 KiB
Go
// Package vfs provides a virtual filing system layer over rclone's
|
|
// native objects.
|
|
//
|
|
// It attempts to behave in a similar way to Go's filing system
|
|
// manipulation code in the os package. The same named function
|
|
// should behave in an identical fashion. The objects also obey Go's
|
|
// standard interfaces.
|
|
//
|
|
// Note that paths don't start or end with /, so the root directory
|
|
// may be referred to as "". However Stat strips slashes so you can
|
|
// use paths with slashes in.
|
|
//
|
|
// It also includes directory caching
|
|
//
|
|
// The vfs package returns Error values to signal precisely which
|
|
// error conditions have ocurred. It may also return general errors
|
|
// it receives. It tries to use os Error values (eg os.ErrExist)
|
|
// where possible.
|
|
package vfs
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/log"
|
|
)
|
|
|
|
// DefaultOpt is the default values uses for Opt
|
|
var DefaultOpt = Options{
|
|
NoModTime: false,
|
|
NoChecksum: false,
|
|
NoSeek: false,
|
|
DirCacheTime: 5 * 60 * time.Second,
|
|
PollInterval: time.Minute,
|
|
ReadOnly: false,
|
|
Umask: 0,
|
|
UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
|
|
GID: ^uint32(0), // overriden for non windows in mount_unix.go
|
|
DirPerms: os.FileMode(0777),
|
|
FilePerms: os.FileMode(0666),
|
|
CacheMode: CacheModeOff,
|
|
CacheMaxAge: 3600 * time.Second,
|
|
CachePollInterval: 60 * time.Second,
|
|
ChunkSize: 128 * fs.MebiByte,
|
|
ChunkSizeLimit: -1,
|
|
CacheMaxSize: -1,
|
|
CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
|
|
}
|
|
|
|
// Node represents either a directory (*Dir) or a file (*File)
|
|
type Node interface {
|
|
os.FileInfo
|
|
IsFile() bool
|
|
Inode() uint64
|
|
SetModTime(modTime time.Time) error
|
|
Sync() error
|
|
Remove() error
|
|
RemoveAll() error
|
|
DirEntry() fs.DirEntry
|
|
VFS() *VFS
|
|
Open(flags int) (Handle, error)
|
|
Truncate(size int64) error
|
|
Path() string
|
|
}
|
|
|
|
// Check interfaces
|
|
var (
|
|
_ Node = (*File)(nil)
|
|
_ Node = (*Dir)(nil)
|
|
)
|
|
|
|
// Nodes is a slice of Node
|
|
type Nodes []Node
|
|
|
|
// Sort functions
|
|
func (ns Nodes) Len() int { return len(ns) }
|
|
func (ns Nodes) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] }
|
|
func (ns Nodes) Less(i, j int) bool { return ns[i].Path() < ns[j].Path() }
|
|
|
|
// Noder represents something which can return a node
|
|
type Noder interface {
|
|
fmt.Stringer
|
|
Node() Node
|
|
}
|
|
|
|
// Check interfaces
|
|
var (
|
|
_ Noder = (*File)(nil)
|
|
_ Noder = (*Dir)(nil)
|
|
_ Noder = (*ReadFileHandle)(nil)
|
|
_ Noder = (*WriteFileHandle)(nil)
|
|
_ Noder = (*RWFileHandle)(nil)
|
|
_ Noder = (*DirHandle)(nil)
|
|
)
|
|
|
|
// OsFiler is the methods on *os.File
|
|
type OsFiler interface {
|
|
Chdir() error
|
|
Chmod(mode os.FileMode) error
|
|
Chown(uid, gid int) error
|
|
Close() error
|
|
Fd() uintptr
|
|
Name() string
|
|
Read(b []byte) (n int, err error)
|
|
ReadAt(b []byte, off int64) (n int, err error)
|
|
Readdir(n int) ([]os.FileInfo, error)
|
|
Readdirnames(n int) (names []string, err error)
|
|
Seek(offset int64, whence int) (ret int64, err error)
|
|
Stat() (os.FileInfo, error)
|
|
Sync() error
|
|
Truncate(size int64) error
|
|
Write(b []byte) (n int, err error)
|
|
WriteAt(b []byte, off int64) (n int, err error)
|
|
WriteString(s string) (n int, err error)
|
|
}
|
|
|
|
// Handle is the interface statisified by open files or directories.
|
|
// It is the methods on *os.File, plus a few more useful for FUSE
|
|
// filingsystems. Not all of them are supported.
|
|
type Handle interface {
|
|
OsFiler
|
|
// Additional methods useful for FUSE filesystems
|
|
Flush() error
|
|
Release() error
|
|
Node() Node
|
|
// Size() int64
|
|
}
|
|
|
|
// baseHandle implements all the missing methods
|
|
type baseHandle struct{}
|
|
|
|
func (h baseHandle) Chdir() error { return ENOSYS }
|
|
func (h baseHandle) Chmod(mode os.FileMode) error { return ENOSYS }
|
|
func (h baseHandle) Chown(uid, gid int) error { return ENOSYS }
|
|
func (h baseHandle) Close() error { return ENOSYS }
|
|
func (h baseHandle) Fd() uintptr { return 0 }
|
|
func (h baseHandle) Name() string { return "" }
|
|
func (h baseHandle) Read(b []byte) (n int, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) ReadAt(b []byte, off int64) (n int, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) Readdir(n int) ([]os.FileInfo, error) { return nil, ENOSYS }
|
|
func (h baseHandle) Readdirnames(n int) (names []string, err error) { return nil, ENOSYS }
|
|
func (h baseHandle) Seek(offset int64, whence int) (ret int64, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) Stat() (os.FileInfo, error) { return nil, ENOSYS }
|
|
func (h baseHandle) Sync() error { return nil }
|
|
func (h baseHandle) Truncate(size int64) error { return ENOSYS }
|
|
func (h baseHandle) Write(b []byte) (n int, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) WriteAt(b []byte, off int64) (n int, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) WriteString(s string) (n int, err error) { return 0, ENOSYS }
|
|
func (h baseHandle) Flush() (err error) { return ENOSYS }
|
|
func (h baseHandle) Release() (err error) { return ENOSYS }
|
|
func (h baseHandle) Node() Node { return nil }
|
|
|
|
//func (h baseHandle) Size() int64 { return 0 }
|
|
|
|
// Check interfaces
|
|
var (
|
|
_ OsFiler = (*os.File)(nil)
|
|
_ Handle = (*baseHandle)(nil)
|
|
_ Handle = (*ReadFileHandle)(nil)
|
|
_ Handle = (*WriteFileHandle)(nil)
|
|
_ Handle = (*DirHandle)(nil)
|
|
)
|
|
|
|
// VFS represents the top level filing system
|
|
type VFS struct {
|
|
f fs.Fs
|
|
root *Dir
|
|
Opt Options
|
|
cache *cache
|
|
cancel context.CancelFunc
|
|
usageMu sync.Mutex
|
|
usageTime time.Time
|
|
usage *fs.Usage
|
|
pollChan chan time.Duration
|
|
}
|
|
|
|
// Options is options for creating the vfs
|
|
type Options struct {
|
|
NoSeek bool // don't allow seeking if set
|
|
NoChecksum bool // don't check checksums if set
|
|
ReadOnly bool // if set VFS is read only
|
|
NoModTime bool // don't read mod times for files
|
|
DirCacheTime time.Duration // how long to consider directory listing cache valid
|
|
PollInterval time.Duration
|
|
Umask int
|
|
UID uint32
|
|
GID uint32
|
|
DirPerms os.FileMode
|
|
FilePerms os.FileMode
|
|
ChunkSize fs.SizeSuffix // if > 0 read files in chunks
|
|
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
|
|
CacheMode CacheMode
|
|
CacheMaxAge time.Duration
|
|
CacheMaxSize fs.SizeSuffix
|
|
CachePollInterval time.Duration
|
|
CaseInsensitive bool
|
|
}
|
|
|
|
// New creates a new VFS and root directory. If opt is nil, then
|
|
// DefaultOpt will be used
|
|
func New(f fs.Fs, opt *Options) *VFS {
|
|
fsDir := fs.NewDir("", time.Now())
|
|
vfs := &VFS{
|
|
f: f,
|
|
}
|
|
|
|
// Make a copy of the options
|
|
if opt != nil {
|
|
vfs.Opt = *opt
|
|
} else {
|
|
vfs.Opt = DefaultOpt
|
|
}
|
|
|
|
// Mask the permissions with the umask
|
|
vfs.Opt.DirPerms &= ^os.FileMode(vfs.Opt.Umask)
|
|
vfs.Opt.FilePerms &= ^os.FileMode(vfs.Opt.Umask)
|
|
|
|
// Make sure directories are returned as directories
|
|
vfs.Opt.DirPerms |= os.ModeDir
|
|
|
|
// Create root directory
|
|
vfs.root = newDir(vfs, f, nil, fsDir)
|
|
|
|
// Start polling function
|
|
if do := vfs.f.Features().ChangeNotify; do != nil {
|
|
vfs.pollChan = make(chan time.Duration)
|
|
do(context.TODO(), vfs.root.changeNotify, vfs.pollChan)
|
|
vfs.pollChan <- vfs.Opt.PollInterval
|
|
} else {
|
|
fs.Infof(f, "poll-interval is not supported by this remote")
|
|
}
|
|
|
|
vfs.SetCacheMode(vfs.Opt.CacheMode)
|
|
|
|
// add the remote control
|
|
vfs.addRC()
|
|
return vfs
|
|
}
|
|
|
|
// Fs returns the Fs passed into the New call
|
|
func (vfs *VFS) Fs() fs.Fs {
|
|
return vfs.f
|
|
}
|
|
|
|
// SetCacheMode change the cache mode
|
|
func (vfs *VFS) SetCacheMode(cacheMode CacheMode) {
|
|
vfs.Shutdown()
|
|
vfs.cache = nil
|
|
if cacheMode > CacheModeOff {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cache, err := newCache(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
|
|
if err != nil {
|
|
fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err)
|
|
vfs.Opt.CacheMode = CacheModeOff
|
|
cancel()
|
|
return
|
|
}
|
|
vfs.Opt.CacheMode = cacheMode
|
|
vfs.cancel = cancel
|
|
vfs.cache = cache
|
|
}
|
|
}
|
|
|
|
// Shutdown stops any background go-routines
|
|
func (vfs *VFS) Shutdown() {
|
|
if vfs.cancel != nil {
|
|
vfs.cancel()
|
|
vfs.cancel = nil
|
|
}
|
|
}
|
|
|
|
// CleanUp deletes the contents of the on disk cache
|
|
func (vfs *VFS) CleanUp() error {
|
|
if vfs.Opt.CacheMode == CacheModeOff {
|
|
return nil
|
|
}
|
|
return vfs.cache.cleanUp()
|
|
}
|
|
|
|
// FlushDirCache empties the directory cache
|
|
func (vfs *VFS) FlushDirCache() {
|
|
vfs.root.ForgetAll()
|
|
}
|
|
|
|
// WaitForWriters sleeps until all writers have finished or
|
|
// time.Duration has elapsed
|
|
func (vfs *VFS) WaitForWriters(timeout time.Duration) {
|
|
defer log.Trace(nil, "timeout=%v", timeout)("")
|
|
const tickTime = 1 * time.Second
|
|
deadline := time.NewTimer(timeout)
|
|
defer deadline.Stop()
|
|
tick := time.NewTimer(tickTime)
|
|
defer tick.Stop()
|
|
tick.Stop()
|
|
for {
|
|
writers := 0
|
|
vfs.root.walk(func(d *Dir) {
|
|
fs.Debugf(d.path, "Looking for writers")
|
|
// NB d.mu is held by walk() here
|
|
for leaf, item := range d.items {
|
|
fs.Debugf(leaf, "reading active writers")
|
|
if file, ok := item.(*File); ok {
|
|
n := file.activeWriters()
|
|
if n != 0 {
|
|
fs.Debugf(file, "active writers %d", n)
|
|
}
|
|
writers += n
|
|
}
|
|
}
|
|
})
|
|
if writers == 0 {
|
|
return
|
|
}
|
|
fs.Debugf(nil, "Still %d writers active, waiting %v", writers, tickTime)
|
|
tick.Reset(tickTime)
|
|
select {
|
|
case <-tick.C:
|
|
break
|
|
case <-deadline.C:
|
|
fs.Errorf(nil, "Exiting even though %d writers are active after %v", writers, timeout)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Root returns the root node
|
|
func (vfs *VFS) Root() (*Dir, error) {
|
|
// fs.Debugf(vfs.f, "Root()")
|
|
return vfs.root, nil
|
|
}
|
|
|
|
var inodeCount uint64
|
|
|
|
// newInode creates a new unique inode number
|
|
func newInode() (inode uint64) {
|
|
return atomic.AddUint64(&inodeCount, 1)
|
|
}
|
|
|
|
// Stat finds the Node by path starting from the root
|
|
//
|
|
// It is the equivalent of os.Stat - Node contains the os.FileInfo
|
|
// interface.
|
|
func (vfs *VFS) Stat(path string) (node Node, err error) {
|
|
path = strings.Trim(path, "/")
|
|
node = vfs.root
|
|
for path != "" {
|
|
i := strings.IndexRune(path, '/')
|
|
var name string
|
|
if i < 0 {
|
|
name, path = path, ""
|
|
} else {
|
|
name, path = path[:i], path[i+1:]
|
|
}
|
|
if name == "" {
|
|
continue
|
|
}
|
|
dir, ok := node.(*Dir)
|
|
if !ok {
|
|
// We need to look in a directory, but found a file
|
|
return nil, ENOENT
|
|
}
|
|
node, err = dir.Stat(name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// StatParent finds the parent directory and the leaf name of a path
|
|
func (vfs *VFS) StatParent(name string) (dir *Dir, leaf string, err error) {
|
|
name = strings.Trim(name, "/")
|
|
parent, leaf := path.Split(name)
|
|
node, err := vfs.Stat(parent)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if node.IsFile() {
|
|
return nil, "", os.ErrExist
|
|
}
|
|
dir = node.(*Dir)
|
|
return dir, leaf, nil
|
|
}
|
|
|
|
// decodeOpenFlags returns a string representing the open flags
|
|
func decodeOpenFlags(flags int) string {
|
|
var out []string
|
|
rdwrMode := flags & accessModeMask
|
|
switch rdwrMode {
|
|
case os.O_RDONLY:
|
|
out = append(out, "O_RDONLY")
|
|
case os.O_WRONLY:
|
|
out = append(out, "O_WRONLY")
|
|
case os.O_RDWR:
|
|
out = append(out, "O_RDWR")
|
|
default:
|
|
out = append(out, fmt.Sprintf("0x%X", rdwrMode))
|
|
}
|
|
if flags&os.O_APPEND != 0 {
|
|
out = append(out, "O_APPEND")
|
|
}
|
|
if flags&os.O_CREATE != 0 {
|
|
out = append(out, "O_CREATE")
|
|
}
|
|
if flags&os.O_EXCL != 0 {
|
|
out = append(out, "O_EXCL")
|
|
}
|
|
if flags&os.O_SYNC != 0 {
|
|
out = append(out, "O_SYNC")
|
|
}
|
|
if flags&os.O_TRUNC != 0 {
|
|
out = append(out, "O_TRUNC")
|
|
}
|
|
flags &^= accessModeMask | os.O_APPEND | os.O_CREATE | os.O_EXCL | os.O_SYNC | os.O_TRUNC
|
|
if flags != 0 {
|
|
out = append(out, fmt.Sprintf("0x%X", flags))
|
|
}
|
|
return strings.Join(out, "|")
|
|
}
|
|
|
|
// OpenFile a file according to the flags and perm provided
|
|
func (vfs *VFS) OpenFile(name string, flags int, perm os.FileMode) (fd Handle, err error) {
|
|
defer log.Trace(name, "flags=%s, perm=%v", decodeOpenFlags(flags), perm)("fd=%v, err=%v", &fd, &err)
|
|
|
|
// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html
|
|
// The result of using O_TRUNC with O_RDONLY is undefined.
|
|
// Linux seems to truncate the file, but we prefer to return EINVAL
|
|
if flags&accessModeMask == os.O_RDONLY && flags&os.O_TRUNC != 0 {
|
|
return nil, EINVAL
|
|
}
|
|
|
|
node, err := vfs.Stat(name)
|
|
if err != nil {
|
|
if err != ENOENT || flags&os.O_CREATE == 0 {
|
|
return nil, err
|
|
}
|
|
// If not found and O_CREATE then create the file
|
|
dir, leaf, err := vfs.StatParent(name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
node, err = dir.Create(leaf, flags)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return node.Open(flags)
|
|
}
|
|
|
|
// Rename oldName to newName
|
|
func (vfs *VFS) Rename(oldName, newName string) error {
|
|
// find the parent directories
|
|
oldDir, oldLeaf, err := vfs.StatParent(oldName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
newDir, newLeaf, err := vfs.StatParent(newName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = oldDir.Rename(oldLeaf, newLeaf, newDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Statfs returns into about the filing system if known
|
|
//
|
|
// The values will be -1 if they aren't known
|
|
//
|
|
// This information is cached for the DirCacheTime interval
|
|
func (vfs *VFS) Statfs() (total, used, free int64) {
|
|
// defer log.Trace("/", "")("total=%d, used=%d, free=%d", &total, &used, &free)
|
|
vfs.usageMu.Lock()
|
|
defer vfs.usageMu.Unlock()
|
|
total, used, free = -1, -1, -1
|
|
doAbout := vfs.f.Features().About
|
|
if doAbout == nil {
|
|
return
|
|
}
|
|
if vfs.usageTime.IsZero() || time.Since(vfs.usageTime) >= vfs.Opt.DirCacheTime {
|
|
var err error
|
|
vfs.usage, err = doAbout(context.TODO())
|
|
vfs.usageTime = time.Now()
|
|
if err != nil {
|
|
fs.Errorf(vfs.f, "Statfs failed: %v", err)
|
|
return
|
|
}
|
|
}
|
|
if u := vfs.usage; u != nil {
|
|
if u.Total != nil {
|
|
total = *u.Total
|
|
}
|
|
if u.Free != nil {
|
|
free = *u.Free
|
|
}
|
|
if u.Used != nil {
|
|
used = *u.Used
|
|
}
|
|
}
|
|
return
|
|
}
|