forked from TrueCloudLab/rclone
union: Implement policy by least number of object
This commit is contained in:
parent
c4545465e7
commit
7da83346bf
5 changed files with 175 additions and 1 deletions
116
backend/union/policy/eplno.go
Normal file
116
backend/union/policy/eplno.go
Normal file
|
@ -0,0 +1,116 @@
|
|||
package policy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
|
||||
"github.com/rclone/rclone/backend/union/upstream"
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerPolicy("eplno", &EpLno{})
|
||||
}
|
||||
|
||||
// EpLno stands for existing path, least number of objects
|
||||
// Of all the candidates on which the path exists choose the one with the least number of objects
|
||||
type EpLno struct {
|
||||
EpAll
|
||||
}
|
||||
|
||||
func (p *EpLno) lno(upstreams []*upstream.Fs) (*upstream.Fs, error) {
|
||||
var minNumObj int64 = math.MaxInt64
|
||||
var lnoUpstream *upstream.Fs
|
||||
for _, u := range upstreams {
|
||||
numObj, err := u.GetNumObjects()
|
||||
if err != nil {
|
||||
fs.LogPrintf(fs.LogLevelNotice, nil,
|
||||
"Number of Objects is not supported for upstream %s, treating as 0", u.Name())
|
||||
}
|
||||
if minNumObj > numObj {
|
||||
minNumObj = numObj
|
||||
lnoUpstream = u
|
||||
}
|
||||
}
|
||||
if lnoUpstream == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return lnoUpstream, nil
|
||||
}
|
||||
|
||||
func (p *EpLno) lnoEntries(entries []upstream.Entry) (upstream.Entry, error) {
|
||||
var minNumObj int64 = math.MaxInt64
|
||||
var lnoEntry upstream.Entry
|
||||
for _, e := range entries {
|
||||
numObj, err := e.UpstreamFs().GetNumObjects()
|
||||
if err != nil {
|
||||
fs.LogPrintf(fs.LogLevelNotice, nil,
|
||||
"Number of Objects is not supported for upstream %s, treating as 0", e.UpstreamFs().Name())
|
||||
}
|
||||
if minNumObj > numObj {
|
||||
minNumObj = numObj
|
||||
lnoEntry = e
|
||||
}
|
||||
}
|
||||
return lnoEntry, nil
|
||||
}
|
||||
|
||||
// Action category policy, governing the modification of files and directories
|
||||
func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
|
||||
upstreams, err := p.EpAll.Action(ctx, upstreams, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u, err := p.lno(upstreams)
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := p.lnoEntries(entries)
|
||||
return []upstream.Entry{e}, err
|
||||
}
|
||||
|
||||
// Create category policy, governing the creation of files and directories
|
||||
func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
|
||||
upstreams, err := p.EpAll.Create(ctx, upstreams, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u, err := p.lno(upstreams)
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := p.lnoEntries(entries)
|
||||
return []upstream.Entry{e}, err
|
||||
}
|
||||
|
||||
// Search category policy, governing the access to files and directories
|
||||
func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
|
||||
if len(upstreams) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
upstreams, err := p.epall(ctx, upstreams, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p.lno(upstreams)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return p.lnoEntries(entries)
|
||||
}
|
33
backend/union/policy/lno.go
Normal file
33
backend/union/policy/lno.go
Normal file
|
@ -0,0 +1,33 @@
|
|||
package policy
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/rclone/rclone/backend/union/upstream"
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerPolicy("lno", &Lno{})
|
||||
}
|
||||
|
||||
// Lno stands for least number of objects
|
||||
// Search category: same as eplno.
|
||||
// Action category: same as eplno.
|
||||
// Create category: Pick the drive with the least number of objects.
|
||||
type Lno struct {
|
||||
EpLno
|
||||
}
|
||||
|
||||
// Create category policy, governing the creation of files and directories
|
||||
func (p *Lno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
|
||||
if len(upstreams) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
upstreams = filterNC(upstreams)
|
||||
if len(upstreams) == 0 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
u, err := p.lno(upstreams)
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
|
@ -11,7 +11,7 @@ func init() {
|
|||
registerPolicy("lus", &Lus{})
|
||||
}
|
||||
|
||||
// Lus stands for most free space
|
||||
// Lus stands for least free space
|
||||
// Search category: same as eplus.
|
||||
// Action category: same as eplus.
|
||||
// Create category: Pick the drive with the least used space.
|
||||
|
|
|
@ -182,6 +182,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
|||
if f.usage.Free != nil {
|
||||
*f.usage.Free -= size
|
||||
}
|
||||
if f.usage.Objects != nil {
|
||||
*f.usage.Objects++
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
@ -208,6 +211,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
|||
if f.usage.Free != nil {
|
||||
*f.usage.Free -= size
|
||||
}
|
||||
if f.usage.Objects != nil {
|
||||
*f.usage.Objects++
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
@ -282,6 +288,22 @@ func (f *Fs) GetUsedSpace() (int64, error) {
|
|||
return *f.usage.Used, nil
|
||||
}
|
||||
|
||||
// GetNumObjects get the number of objects of the fs
|
||||
func (f *Fs) GetNumObjects() (int64, error) {
|
||||
if atomic.LoadInt64(&f.cacheExpiry) <= time.Now().Unix() {
|
||||
err := f.updateUsage()
|
||||
if err != nil {
|
||||
return 0, ErrUsageFieldNotSupported
|
||||
}
|
||||
}
|
||||
f.cacheMutex.RLock()
|
||||
defer f.cacheMutex.RUnlock()
|
||||
if f.usage.Objects == nil {
|
||||
return 0, ErrUsageFieldNotSupported
|
||||
}
|
||||
return *f.usage.Objects, nil
|
||||
}
|
||||
|
||||
func (f *Fs) updateUsage() (err error) {
|
||||
if do := f.RootFs.Features().About; do == nil {
|
||||
return ErrUsageFieldNotSupported
|
||||
|
|
|
@ -58,6 +58,7 @@ Some policies rely on quota information. These policies should be used only if y
|
|||
| lfs, eplfs | Free |
|
||||
| mfs, epmfs | Free |
|
||||
| lus, eplus | Used |
|
||||
| lno, eplno | Objects |
|
||||
|
||||
To check if your upstream support the field, run `rclone about remote: [flags]` and see if the reuqired field exists.
|
||||
|
||||
|
@ -82,11 +83,13 @@ THe policies definition are inspired by [trapexit/mergerfs](https://github.com/t
|
|||
| epff (existing path, first found) | Act on the first one found, by the time upstreams reply, where the relative path exists. |
|
||||
| eplfs (existing path, least free space) | Of all the remotes on which the relative path exists choose the one with the least free space. |
|
||||
| eplus (existing path, least used space) | Of all the remotes on which the relative path exists choose the one with the least used space. |
|
||||
| eplno (existing path, least number of objects) | Of all the remotes on which the relative path exists choose the one with the least number of objects. |
|
||||
| epmfs (existing path, most free space) | Of all the remotes on which the relative path exists choose the one with the most free space. |
|
||||
| eprand (existing path, random) | Calls **epall** and then randomizes. Returns only one remote. |
|
||||
| ff (first found) | Search category: same as **epff**. Action category: same as **epff**. Create category: Act on the first one found by the time upstreams reply. |
|
||||
| lfs (least free space) | Search category: same as **eplfs**. Action category: same as **eplfs**. Create category: Pick the remote with the least available free space. |
|
||||
| lus (least used space) | Search category: same as **eplus**. Action category: same as **eplus**. Create category: Pick the remote with the least used space. |
|
||||
| lno (least number of objects) | Search category: same as **eplno**. Action category: same as **eplno**. Create category: Pick the remote with the least number of objects. |
|
||||
| mfs (most free space) | Search category: same as **epmfs**. Action category: same as **epmfs**. Create category: Pick the remote with the most available free space. |
|
||||
| newest | Pick the file / directory with the largest mtime. |
|
||||
| rand (random) | Calls **all** and then randomizes. Returns only one remote. |
|
||||
|
|
Loading…
Add table
Reference in a new issue