alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

union: change epff policy to search local disks first

Browse source 
Its always been random which remote epff/ff finds first. Make it so
that we search local disks first which will save on network resources.

See: https://forum.rclone.org/t/rclone-union-no-longer-preferring-local-copies-windows/32002/3
This commit is contained in:
Nick Craig-Wood 2022-07-21 17:23:56 +01:00
parent 7a24c173f6
commit 1d1d847f18
2 changed files with 24 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
backend/union/policy/epff.go
View file

@ -16,11 +16,14 @@ func init() {
// Given the order of the candidates, act on the first one found where the relative path exists. // Given the order of the candidates, act on the first one found where the relative path exists.
type EpFF struct{} type EpFF struct{}
func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath string) (*upstream.Fs, error) { func (p *EpFF) epffIsLocal(ctx context.Context, upstreams []*upstream.Fs, filePath string, isLocal bool) (*upstream.Fs, error) {
ch := make(chan *upstream.Fs, len(upstreams)) ch := make(chan *upstream.Fs, len(upstreams))
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
for _, u := range upstreams { for _, u := range upstreams {
if u.IsLocal() != isLocal {
continue
}
u := u // Closure u := u // Closure
go func() { go func() {
rfs := u.RootFs rfs := u.RootFs
@ -32,7 +35,10 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath stri
}() }()
} }
var u *upstream.Fs var u *upstream.Fs
for range upstreams { for _, upstream := range upstreams {
if upstream.IsLocal() != isLocal {
continue
}
u = <-ch u = <-ch
if u != nil { if u != nil {
break break
@ -44,6 +50,15 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath stri
return u, nil return u, nil
} }
func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath string) (*upstream.Fs, error) {
// search local disks first
u, err := p.epffIsLocal(ctx, upstreams, filePath, true)
if err == fs.ErrorObjectNotFound {
u, err = p.epffIsLocal(ctx, upstreams, filePath, false)
}
return u, err
}
// Action category policy, governing the modification of files and directories // Action category policy, governing the modification of files and directories
func (p *EpFF) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) { func (p *EpFF) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
if len(upstreams) == 0 { if len(upstreams) == 0 {

7
backend/union/upstream/upstream.go
View file

@ -34,6 +34,7 @@ type Fs struct {
Opt *common.Options Opt *common.Options
writable bool writable bool
creatable bool creatable bool
isLocal bool
usage *fs.Usage // Cache the usage usage *fs.Usage // Cache the usage
cacheTime time.Duration // cache duration cacheTime time.Duration // cache duration
cacheMutex sync.RWMutex cacheMutex sync.RWMutex
@ -95,6 +96,7 @@ func New(ctx context.Context, remote, root string, opt *common.Options) (*Fs, er
return nil, err return nil, err
} }
f.RootFs = rFs f.RootFs = rFs
f.isLocal = rFs.Features().IsLocal
rootString := fspath.JoinRootPath(remote, root) rootString := fspath.JoinRootPath(remote, root)
myFs, err := cache.Get(ctx, rootString) myFs, err := cache.Get(ctx, rootString)
if err != nil && err != fs.ErrorIsFile { if err != nil && err != fs.ErrorIsFile {
@ -142,6 +144,11 @@ func (f *Fs) WrapEntry(e fs.DirEntry) (Entry, error) {
} }
} }
// IsLocal true if the upstream Fs is a local disk
func (f *Fs) IsLocal() bool {
return f.isLocal
}
// UpstreamFs get the upstream Fs the entry is stored in // UpstreamFs get the upstream Fs the entry is stored in
func (e *Directory) UpstreamFs() *Fs { func (e *Directory) UpstreamFs() *Fs {
return e.f return e.f