diff --git a/backend/union/policy/eplno.go b/backend/union/policy/eplno.go new file mode 100644 index 000000000..b81f85c31 --- /dev/null +++ b/backend/union/policy/eplno.go @@ -0,0 +1,116 @@ +package policy + +import ( + "context" + "math" + + "github.com/rclone/rclone/backend/union/upstream" + "github.com/rclone/rclone/fs" +) + +func init() { + registerPolicy("eplno", &EpLno{}) +} + +// EpLno stands for existing path, least number of objects +// Of all the candidates on which the path exists choose the one with the least number of objects +type EpLno struct { + EpAll +} + +func (p *EpLno) lno(upstreams []*upstream.Fs) (*upstream.Fs, error) { + var minNumObj int64 = math.MaxInt64 + var lnoUpstream *upstream.Fs + for _, u := range upstreams { + numObj, err := u.GetNumObjects() + if err != nil { + fs.LogPrintf(fs.LogLevelNotice, nil, + "Number of Objects is not supported for upstream %s, treating as 0", u.Name()) + } + if minNumObj > numObj { + minNumObj = numObj + lnoUpstream = u + } + } + if lnoUpstream == nil { + return nil, fs.ErrorObjectNotFound + } + return lnoUpstream, nil +} + +func (p *EpLno) lnoEntries(entries []upstream.Entry) (upstream.Entry, error) { + var minNumObj int64 = math.MaxInt64 + var lnoEntry upstream.Entry + for _, e := range entries { + numObj, err := e.UpstreamFs().GetNumObjects() + if err != nil { + fs.LogPrintf(fs.LogLevelNotice, nil, + "Number of Objects is not supported for upstream %s, treating as 0", e.UpstreamFs().Name()) + } + if minNumObj > numObj { + minNumObj = numObj + lnoEntry = e + } + } + return lnoEntry, nil +} + +// Action category policy, governing the modification of files and directories +func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) { + upstreams, err := p.EpAll.Action(ctx, upstreams, path) + if err != nil { + return nil, err + } + u, err := p.lno(upstreams) + return []*upstream.Fs{u}, err +} + +// ActionEntries is ACTION category policy but receving a set of candidate entries +func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) { + entries, err := p.EpAll.ActionEntries(entries...) + if err != nil { + return nil, err + } + e, err := p.lnoEntries(entries) + return []upstream.Entry{e}, err +} + +// Create category policy, governing the creation of files and directories +func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) { + upstreams, err := p.EpAll.Create(ctx, upstreams, path) + if err != nil { + return nil, err + } + u, err := p.lno(upstreams) + return []*upstream.Fs{u}, err +} + +// CreateEntries is CREATE category policy but receving a set of candidate entries +func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) { + entries, err := p.EpAll.CreateEntries(entries...) + if err != nil { + return nil, err + } + e, err := p.lnoEntries(entries) + return []upstream.Entry{e}, err +} + +// Search category policy, governing the access to files and directories +func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) { + if len(upstreams) == 0 { + return nil, fs.ErrorObjectNotFound + } + upstreams, err := p.epall(ctx, upstreams, path) + if err != nil { + return nil, err + } + return p.lno(upstreams) +} + +// SearchEntries is SEARCH category policy but receving a set of candidate entries +func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) { + if len(entries) == 0 { + return nil, fs.ErrorObjectNotFound + } + return p.lnoEntries(entries) +} diff --git a/backend/union/policy/lno.go b/backend/union/policy/lno.go new file mode 100644 index 000000000..7d9a99d8a --- /dev/null +++ b/backend/union/policy/lno.go @@ -0,0 +1,33 @@ +package policy + +import ( + "context" + + "github.com/rclone/rclone/backend/union/upstream" + "github.com/rclone/rclone/fs" +) + +func init() { + registerPolicy("lno", &Lno{}) +} + +// Lno stands for least number of objects +// Search category: same as eplno. +// Action category: same as eplno. +// Create category: Pick the drive with the least number of objects. +type Lno struct { + EpLno +} + +// Create category policy, governing the creation of files and directories +func (p *Lno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) { + if len(upstreams) == 0 { + return nil, fs.ErrorObjectNotFound + } + upstreams = filterNC(upstreams) + if len(upstreams) == 0 { + return nil, fs.ErrorPermissionDenied + } + u, err := p.lno(upstreams) + return []*upstream.Fs{u}, err +} diff --git a/backend/union/policy/lus.go b/backend/union/policy/lus.go index ad9b3fc77..4846e6dc8 100644 --- a/backend/union/policy/lus.go +++ b/backend/union/policy/lus.go @@ -11,7 +11,7 @@ func init() { registerPolicy("lus", &Lus{}) } -// Lus stands for most free space +// Lus stands for least free space // Search category: same as eplus. // Action category: same as eplus. // Create category: Pick the drive with the least used space. diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 07ec75661..4538f6685 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -182,6 +182,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . if f.usage.Free != nil { *f.usage.Free -= size } + if f.usage.Objects != nil { + *f.usage.Objects++ + } return o, nil } @@ -208,6 +211,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt if f.usage.Free != nil { *f.usage.Free -= size } + if f.usage.Objects != nil { + *f.usage.Objects++ + } return o, nil } @@ -282,6 +288,22 @@ func (f *Fs) GetUsedSpace() (int64, error) { return *f.usage.Used, nil } +// GetNumObjects get the number of objects of the fs +func (f *Fs) GetNumObjects() (int64, error) { + if atomic.LoadInt64(&f.cacheExpiry) <= time.Now().Unix() { + err := f.updateUsage() + if err != nil { + return 0, ErrUsageFieldNotSupported + } + } + f.cacheMutex.RLock() + defer f.cacheMutex.RUnlock() + if f.usage.Objects == nil { + return 0, ErrUsageFieldNotSupported + } + return *f.usage.Objects, nil +} + func (f *Fs) updateUsage() (err error) { if do := f.RootFs.Features().About; do == nil { return ErrUsageFieldNotSupported diff --git a/docs/content/union.md b/docs/content/union.md index dac65134e..a3e7b4291 100644 --- a/docs/content/union.md +++ b/docs/content/union.md @@ -58,6 +58,7 @@ Some policies rely on quota information. These policies should be used only if y | lfs, eplfs | Free | | mfs, epmfs | Free | | lus, eplus | Used | +| lno, eplno | Objects | To check if your upstream support the field, run `rclone about remote: [flags]` and see if the reuqired field exists. @@ -82,11 +83,13 @@ THe policies definition are inspired by [trapexit/mergerfs](https://github.com/t | epff (existing path, first found) | Act on the first one found, by the time upstreams reply, where the relative path exists. | | eplfs (existing path, least free space) | Of all the remotes on which the relative path exists choose the one with the least free space. | | eplus (existing path, least used space) | Of all the remotes on which the relative path exists choose the one with the least used space. | +| eplno (existing path, least number of objects) | Of all the remotes on which the relative path exists choose the one with the least number of objects. | | epmfs (existing path, most free space) | Of all the remotes on which the relative path exists choose the one with the most free space. | | eprand (existing path, random) | Calls **epall** and then randomizes. Returns only one remote. | | ff (first found) | Search category: same as **epff**. Action category: same as **epff**. Create category: Act on the first one found by the time upstreams reply. | | lfs (least free space) | Search category: same as **eplfs**. Action category: same as **eplfs**. Create category: Pick the remote with the least available free space. | | lus (least used space) | Search category: same as **eplus**. Action category: same as **eplus**. Create category: Pick the remote with the least used space. | +| lno (least number of objects) | Search category: same as **eplno**. Action category: same as **eplno**. Create category: Pick the remote with the least number of objects. | | mfs (most free space) | Search category: same as **epmfs**. Action category: same as **epmfs**. Create category: Pick the remote with the most available free space. | | newest | Pick the file / directory with the largest mtime. | | rand (random) | Calls **all** and then randomizes. Returns only one remote. |