TrueCloudLab/rclone
16
0
Fork 4
Code Issues 1 Pull requests Projects Releases 4 Packages Wiki Activity Actions

cache: add support for polling

Browse source
This commit is contained in:
remusb 2018-02-10 22:01:05 +02:00 • committed by GitHub
parent 8a25ca786c
commit b33e3f779c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 351 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

151
backend/cache/cache.go vendored
View file

@ -174,6 +174,7 @@ type Fs struct {
plexConnector *plexConnector
backgroundRunner *backgroundWriter
cleanupChan chan bool
parentsForgetFn []func(string)
}
// parseRootPath returns a cleaned root path and a nil error or "" and an error when the path is invalid
@ -380,24 +381,16 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
}
}()
// TODO: Explore something here but now it's not something we want
// when writing from cache, source FS will send a notification and clear it out immediately
//setup dir notification
//doDirChangeNotify := wrappedFs.Features().DirChangeNotify
//if doDirChangeNotify != nil {
// doDirChangeNotify(func(dir string) {
// d := NewAbsDirectory(f, dir)
// d.Flush()
// fs.Infof(dir, "updated from notification")
// }, time.Second * 10)
//}
if doDirChangeNotify := wrappedFs.Features().DirChangeNotify; doDirChangeNotify != nil {
doDirChangeNotify(f.receiveDirChangeNotify, f.chunkCleanInterval)
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
DuplicateFiles: false, // storage doesn't permit this
DirChangeNotify: nil,
}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
// override only those features that use a temp fs and it doesn't support them
f.features.DirChangeNotify = f.DirChangeNotify
if f.tempWritePath != "" {
if f.tempFs.Features().Copy == nil {
f.features.Copy = nil
@ -421,6 +414,81 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
return f, fsErr
}
func (f *Fs) receiveDirChangeNotify(forgetPath string) {
fs.Debugf(f, "notify: expiring cache for '%v'", forgetPath)
// notify upstreams too (vfs)
f.notifyDirChange(forgetPath)
var cd *Directory
co := NewObject(f, forgetPath)
err := f.cache.GetObject(co)
if err == nil {
cd = NewDirectory(f, cleanPath(path.Dir(co.Remote())))
} else {
cd = NewDirectory(f, forgetPath)
}
// we list all the cached objects and expire all of them
entries, err := f.cache.GetDirEntries(cd)
if err != nil {
fs.Debugf(forgetPath, "notify: ignoring notification on non cached dir")
return
}
for i := 0; i < len(entries); i++ {
if co, ok := entries[i].(*Object); ok {
co.CacheTs = time.Now().Add(f.fileAge * -1)
err = f.cache.AddObject(co)
if err != nil {
fs.Errorf(forgetPath, "notify: error expiring '%v': %v", co, err)
} else {
fs.Debugf(forgetPath, "notify: expired %v", co)
}
}
}
// finally, we expire the dir as well
err = f.cache.ExpireDir(cd)
if err != nil {
fs.Errorf(forgetPath, "notify: error expiring '%v': %v", cd, err)
} else {
fs.Debugf(forgetPath, "notify: expired '%v'", cd)
}
}
// notifyDirChange takes a remote (can be dir or entry) and
// tries to determine which is it and notify upstreams of the dir change
func (f *Fs) notifyDirChange(remote string) {
var cd *Directory
co := NewObject(f, remote)
err := f.cache.GetObject(co)
if err == nil {
pd := cleanPath(path.Dir(remote))
cd = NewDirectory(f, pd)
} else {
cd = NewDirectory(f, remote)
}
f.notifyDirChangeUpstream(cd.Remote())
}
// notifyDirChangeUpstream will loop through all the upstreams and notify
// of the provided remote (should be only a dir)
func (f *Fs) notifyDirChangeUpstream(remote string) {
if len(f.parentsForgetFn) > 0 {
for _, fn := range f.parentsForgetFn {
fn(remote)
}
}
}
// DirChangeNotify can subsribe multiple callers
// this is coupled with the wrapped fs DirChangeNotify (if it supports it)
// and also notifies other caches (i.e VFS) to clear out whenever something changes
func (f *Fs) DirChangeNotify(notifyFunc func(string), pollInterval time.Duration) chan bool {
fs.Debugf(f, "subscribing to DirChangeNotify")
f.parentsForgetFn = append(f.parentsForgetFn, notifyFunc)
return make(chan bool)
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
@ -683,6 +751,10 @@ func (f *Fs) Mkdir(dir string) error {
} else {
fs.Infof(parentCd, "mkdir: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
return nil
}
@ -751,6 +823,10 @@ func (f *Fs) Rmdir(dir string) error {
} else {
fs.Infof(parentCd, "rmdir: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
return nil
}
@ -847,6 +923,10 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
} else {
fs.Debugf(srcParent, "dirmove: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(srcParent.Remote())
}
// expire parent dir at the destination path
dstParent := NewDirectory(f, cleanPath(path.Dir(dstRemote)))
@ -856,6 +936,10 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
} else {
fs.Debugf(dstParent, "dirmove: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(dstParent.Remote())
}
// TODO: precache dst dir and save the chunks
return nil
@ -978,6 +1062,10 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
} else {
fs.Infof(parentCd, "put: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
return cachedObj, nil
}
@ -1066,6 +1154,10 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
} else {
fs.Infof(parentCd, "copy: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
// expire src parent
srcParent := NewDirectory(f, cleanPath(path.Dir(src.Remote())))
err = f.cache.ExpireDir(srcParent)
@ -1074,6 +1166,10 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
} else {
fs.Infof(srcParent, "copy: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(srcParent.Remote())
}
return co, nil
}
@ -1158,6 +1254,10 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
} else {
fs.Infof(parentCd, "move: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
// persist new
cachedObj := ObjectFromOriginal(f, obj).persist()
fs.Debugf(cachedObj, "move: added to cache")
@ -1169,6 +1269,10 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
} else {
fs.Infof(parentCd, "move: cache expired")
}
// advertise to DirChangeNotify if wrapped doesn't do that
if f.Fs.Features().DirChangeNotify == nil {
f.notifyDirChangeUpstream(parentCd.Remote())
}
return cachedObj, nil
}
@ -1321,15 +1425,16 @@ func cleanPath(p string) string {
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.PutUncheckeder = (*Fs)(nil)
_ fs.PutStreamer = (*Fs)(nil)
_ fs.CleanUpper = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.Wrapper = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.PutUncheckeder = (*Fs)(nil)
_ fs.PutStreamer = (*Fs)(nil)
_ fs.CleanUpper = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.Wrapper = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.DirChangeNotifier = (*Fs)(nil)
)