diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index f8980c360..863c21050 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -29,7 +29,8 @@ import ( // // Cache may call into Item but care is needed if Item calls Cache -// FIXME size in cache needs to be size on disk if we have sparse files... +// FIXME need to purge cache nodes which don't have backing files and aren't dirty +// these may get created by the VFS layer or may be orphans from reload() // Cache opened files type Cache struct { @@ -97,7 +98,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, er } // load in the cache and metadata off disk - err = c.reload() + err = c.reload(ctx) if err != nil { return nil, errors.Wrap(err, "failed to load cache") } @@ -356,26 +357,30 @@ func (c *Cache) walk(dir string, fn func(osPath string, fi os.FileInfo, name str } // reload walks the cache loading metadata files -func (c *Cache) reload() error { - err := c.walk(c.root, func(osPath string, fi os.FileInfo, name string) error { - if !fi.IsDir() { - _, _ = c.get(name) +// +// It iterates the files first then metadata trees. It doesn't expect +// to find any new items iterating the metadata but it will clear up +// orphan files. +func (c *Cache) reload(ctx context.Context) error { + for _, dir := range []string{c.root, c.metaRoot} { + err := c.walk(dir, func(osPath string, fi os.FileInfo, name string) error { + if fi.IsDir() { + return nil + } + item, found := c.get(name) + if !found { + err := item.reload(ctx) + if err != nil { + fs.Errorf(name, "vfs cache: failed to reload item: %v", err) + } + } + return nil + }) + if err != nil { + return errors.Wrapf(err, "failed to walk cache %q", dir) } - return nil - }) - if err != nil { - return errors.Wrap(err, "failed to walk cache") } - err = c.walk(c.root, func(osPathMeta string, fi os.FileInfo, name string) error { - if !fi.IsDir() { - _, _ = c.get(name) - } - return nil - }) - if err != nil { - return errors.Wrap(err, "failed to walk meta cache") - } - return err + return nil } // purgeOld gets rid of any files that are over age @@ -491,9 +496,16 @@ func (c *Cache) clean() { // Stats c.mu.Lock() newItems, newUsed := len(c.item), fs.SizeSuffix(c.used) + totalInUse := 0 + for _, item := range c.item { + if item.inUse() { + totalInUse++ + } + } c.mu.Unlock() + uploadsInProgress, uploadsQueued := c.writeback.getStats() - fs.Infof(nil, "Cleaned the cache: objects %d (was %d), total size %v (was %v)", newItems, oldItems, newUsed, oldUsed) + fs.Infof(nil, "Cleaned the cache: objects %d (was %d) in use %d, to upload %d, uploading %d, total size %v (was %v)", newItems, oldItems, totalInUse, uploadsQueued, uploadsInProgress, newUsed, oldUsed) } // cleaner calls clean at regular intervals diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index 5ebf8a48c..5112d1416 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -589,6 +589,32 @@ func (item *Item) Close(storeFn StoreFn) (err error) { return err } +// reload is called with valid items recovered from a cache reload. +// +// it is called before the cache has started so opens will be 0 and +// metaDirty will be false. +func (item *Item) reload(ctx context.Context) error { + item.mu.Lock() + dirty := item.info.Dirty + item.mu.Unlock() + if !dirty { + return nil + } + // see if the object still exists + obj, _ := item.c.fremote.NewObject(ctx, item.name) + // open the file with the object (or nil) + err := item.Open(obj) + if err != nil { + return err + } + // close the file to execute the writeback if needed + err = item.Close(nil) + if err != nil { + return err + } + return nil +} + // check the fingerprint of an object and update the item or delete // the cached file accordingly // diff --git a/vfs/vfscache/writeback.go b/vfs/vfscache/writeback.go index 846f082e2..8573228b5 100644 --- a/vfs/vfscache/writeback.go +++ b/vfs/vfscache/writeback.go @@ -309,3 +309,10 @@ func (wb *writeBack) uploader(ctx context.Context) { } } } + +// return the number of uploads in progress +func (wb *writeBack) getStats() (uploadsInProgress, uploadsQueued int) { + wb.mu.Lock() + defer wb.mu.Unlock() + return wb.uploads, len(wb.items) +} diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go index 54a4d213c..2185673ab 100644 --- a/vfs/vfsflags/vfsflags.go +++ b/vfs/vfsflags/vfsflags.go @@ -35,6 +35,6 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match.") flags.DurationVarP(flagSet, &Opt.WriteWait, "vfs-write-wait", "", Opt.WriteWait, "Time to wait for in-sequence write before giving error.") flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.") - flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-writeback", "", Opt.ReadWait, "Time to writeback files after last use when using cache.") + flags.DurationVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", Opt.WriteBack, "Time to writeback files after last use when using cache.") platformFlags(flagSet) }