// This deals with caching of files locally package vfs import ( "fmt" "os" "path" "path/filepath" "runtime" "sort" "strings" "sync" "time" "github.com/djherbis/times" "github.com/ncw/rclone/fs" "github.com/pkg/errors" "golang.org/x/net/context" ) // CacheMode controls the functionality of the cache type CacheMode byte // CacheMode options const ( CacheModeOff CacheMode = iota // cache nothing - return errors for writes which can't be satisfied CacheModeMinimal // cache only the minimum, eg read/write opens CacheModeWrites // cache all files opened with write intent CacheModeFull // cache all files opened in any mode ) var cacheModeToString = []string{ CacheModeOff: "off", CacheModeMinimal: "minimal", CacheModeWrites: "writes", CacheModeFull: "full", } // String turns a CacheMode into a string func (l CacheMode) String() string { if l >= CacheMode(len(cacheModeToString)) { return fmt.Sprintf("CacheMode(%d)", l) } return cacheModeToString[l] } // Set a CacheMode func (l *CacheMode) Set(s string) error { for n, name := range cacheModeToString { if s != "" && name == s { *l = CacheMode(n) return nil } } return errors.Errorf("Unknown cache mode level %q", s) } // Type of the value func (l *CacheMode) Type() string { return "string" } // cache opened files type cache struct { f fs.Fs // fs for the cache directory opt *Options // vfs Options root string // root of the cache directory itemMu sync.Mutex // protects the next two maps item map[string]*cacheItem // files/directories in the cache } // cacheItem is stored in the item map type cacheItem struct { opens int // number of times file is open atime time.Time // last time file was accessed isFile bool // if this is a file or a directory } // newCacheItem returns an item for the cache func newCacheItem(isFile bool) *cacheItem { return &cacheItem{atime: time.Now(), isFile: isFile} } // newCache creates a new cache heirachy for f // // This starts background goroutines which can be cancelled with the // context passed in. func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) { fRoot := filepath.FromSlash(f.Root()) if runtime.GOOS == "windows" { if strings.HasPrefix(fRoot, `\\?`) { fRoot = fRoot[3:] } fRoot = strings.Replace(fRoot, ":", "", -1) } root := filepath.Join(fs.CacheDir, "vfs", f.Name(), fRoot) fs.Debugf(nil, "vfs cache root is %q", root) f, err := fs.NewFs(root) if err != nil { return nil, errors.Wrap(err, "failed to create cache remote") } c := &cache{ f: f, opt: opt, root: root, item: make(map[string]*cacheItem), } go c.cleaner(ctx) return c, nil } // findParent returns the parent directory of name, or "" for the root func findParent(name string) string { parent := path.Dir(name) if parent == "." || parent == "/" { parent = "" } return parent } // toOSPath turns a remote relative name into an OS path in the cache func (c *cache) toOSPath(name string) string { return filepath.Join(c.root, filepath.FromSlash(name)) } // mkdir makes the directory for name in the cache and returns an os // path for the file func (c *cache) mkdir(name string) (string, error) { parent := findParent(name) leaf := path.Base(name) parentPath := c.toOSPath(parent) err := os.MkdirAll(parentPath, 0700) if err != nil { return "", errors.Wrap(err, "make cache directory failed") } c.cacheDir(parent) return filepath.Join(parentPath, leaf), nil } // _get gets name from the cache or creates a new one // // name should be a remote path not an osPath // // must be called with itemMu held func (c *cache) _get(isFile bool, name string) *cacheItem { item := c.item[name] if item == nil { item = newCacheItem(isFile) c.item[name] = item } return item } // get gets name from the cache or creates a new one // // name should be a remote path not an osPath func (c *cache) get(name string) *cacheItem { c.itemMu.Lock() item := c._get(true, name) c.itemMu.Unlock() return item } // updateTime sets the atime of the name to that passed in if it is // newer than the existing or there isn't an existing time. // // name should be a remote path not an osPath func (c *cache) updateTime(name string, when time.Time) { c.itemMu.Lock() item := c._get(true, name) if when.Sub(item.atime) > 0 { fs.Debugf(name, "updateTime: setting atime to %v", when) item.atime = when } c.itemMu.Unlock() } // _open marks name as open, must be called with the lock held // // name should be a remote path not an osPath func (c *cache) _open(isFile bool, name string) { for { item := c._get(isFile, name) item.opens++ item.atime = time.Now() if name == "" { break } isFile = false name = findParent(name) } } // open marks name as open // // name should be a remote path not an osPath func (c *cache) open(name string) { c.itemMu.Lock() c._open(true, name) c.itemMu.Unlock() } // cacheDir marks a directory and its parents as being in the cache // // name should be a remote path not an osPath func (c *cache) cacheDir(name string) { c.itemMu.Lock() defer c.itemMu.Unlock() for { item := c.item[name] if item != nil { break } c.item[name] = newCacheItem(false) if name == "" { break } name = findParent(name) } } // _close marks name as closed - must be called with the lock held func (c *cache) _close(isFile bool, name string) { for { item := c._get(isFile, name) item.opens-- item.atime = time.Now() if item.opens < 0 { fs.Errorf(name, "cache: double close") } if name == "" { break } isFile = false name = findParent(name) } } // close marks name as closed // // name should be a remote path not an osPath func (c *cache) close(name string) { c.itemMu.Lock() c._close(true, name) c.itemMu.Unlock() } // remove should be called if name is deleted func (c *cache) remove(name string) { osPath := c.toOSPath(name) err := os.Remove(osPath) if err != nil && !os.IsNotExist(err) { fs.Errorf(name, "Failed to remove from cache: %v", err) } else { fs.Debugf(name, "Removed from cache") } } // removeDir should be called if dir is deleted and returns true if // the directory is gone. func (c *cache) removeDir(dir string) bool { osPath := c.toOSPath(dir) err := os.Remove(osPath) if err == nil || os.IsNotExist(err) { if err == nil { fs.Debugf(dir, "Removed empty directory") } return true } if !os.IsExist(err) { fs.Errorf(dir, "Failed to remove cached dir: %v", err) } return false } // cleanUp empties the cache of everything func (c *cache) cleanUp() error { return os.RemoveAll(c.root) } // walk walks the cache calling the function func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error { return filepath.Walk(c.root, func(osPath string, fi os.FileInfo, err error) error { if err != nil { return err } // Find path relative to the cache root name, err := filepath.Rel(c.root, osPath) if err != nil { return errors.Wrap(err, "filepath.Rel failed in walk") } if name == "." { name = "" } // And convert into slashes name = filepath.ToSlash(name) return fn(osPath, fi, name) }) } // updateAtimes walks the cache updating any atimes it finds func (c *cache) updateAtimes() error { return c.walk(func(osPath string, fi os.FileInfo, name string) error { if !fi.IsDir() { // Update the atime with that of the file atime := times.Get(fi).AccessTime() c.updateTime(name, atime) } else { c.cacheDir(name) } return nil }) } // purgeOld gets rid of any files that are over age func (c *cache) purgeOld(maxAge time.Duration) { c._purgeOld(maxAge, c.remove, c.removeDir) } func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), removeDir func(name string) bool) { c.itemMu.Lock() defer c.itemMu.Unlock() cutoff := time.Now().Add(-maxAge) for name, item := range c.item { if item.isFile && item.opens == 0 { // If not locked and access time too long ago - delete the file dt := item.atime.Sub(cutoff) // fs.Debugf(name, "atime=%v cutoff=%v, dt=%v", item.atime, cutoff, dt) if dt < 0 { remove(name) // Remove the entry delete(c.item, name) } } } // now find any empty directories var dirs []string for name, item := range c.item { if !item.isFile && item.opens == 0 { dirs = append(dirs, name) } } // remove empty directories in reverse alphabetical order sort.Strings(dirs) for i := len(dirs) - 1; i >= 0; i-- { dir := dirs[i] // Remove the entry if removeDir(dir) { delete(c.item, dir) } } } // clean empties the cache of stuff if it can func (c *cache) clean() { // Cache may be empty so end _, err := os.Stat(c.root) if os.IsNotExist(err) { return } fs.Debugf(nil, "Cleaning the cache") // first walk the FS to update the atimes err = c.updateAtimes() if err != nil { fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err) } // Now remove any files that are over age and any empty // directories c.purgeOld(c.opt.CacheMaxAge) } // cleaner calls clean at regular intervals // // doesn't return until context is cancelled func (c *cache) cleaner(ctx context.Context) { timer := time.NewTicker(c.opt.CachePollInterval) defer timer.Stop() for { select { case <-timer.C: c.clean() case <-ctx.Done(): fs.Debugf(nil, "cache cleaner exiting") return } } }