548 lines
14 KiB
Go
548 lines
14 KiB
Go
package vfs
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/cache"
|
|
"github.com/rclone/rclone/fs/rc"
|
|
"github.com/rclone/rclone/vfs/vfscache/writeback"
|
|
)
|
|
|
|
const getVFSHelp = `
|
|
This command takes an "fs" parameter. If this parameter is not
|
|
supplied and if there is only one VFS in use then that VFS will be
|
|
used. If there is more than one VFS in use then the "fs" parameter
|
|
must be supplied.`
|
|
|
|
// GetVFS gets a VFS with config name "fs" from the cache or returns an error.
|
|
//
|
|
// If "fs" is not set and there is one and only one VFS in the active
|
|
// cache then it returns it. This is for backwards compatibility.
|
|
//
|
|
// This deletes the "fs" parameter from in if it is valid
|
|
func getVFS(in rc.Params) (vfs *VFS, err error) {
|
|
fsString, err := in.GetString("fs")
|
|
if rc.IsErrParamNotFound(err) {
|
|
var count int
|
|
vfs, count = activeCacheEntries()
|
|
if count == 1 {
|
|
return vfs, nil
|
|
} else if count == 0 {
|
|
return nil, errors.New(`no VFS active and "fs" parameter not supplied`)
|
|
}
|
|
return nil, errors.New(`more than one VFS active - need "fs" parameter`)
|
|
} else if err != nil {
|
|
return nil, err
|
|
}
|
|
activeMu.Lock()
|
|
defer activeMu.Unlock()
|
|
fsString = cache.Canonicalize(fsString)
|
|
activeVFS := active[fsString]
|
|
if len(activeVFS) == 0 {
|
|
return nil, fmt.Errorf("no VFS found with name %q", fsString)
|
|
} else if len(activeVFS) > 1 {
|
|
return nil, fmt.Errorf("more than one VFS active with name %q", fsString)
|
|
}
|
|
delete(in, "fs") // delete the fs parameter
|
|
return activeVFS[0], nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/refresh",
|
|
Fn: rcRefresh,
|
|
Title: "Refresh the directory cache.",
|
|
Help: `
|
|
This reads the directories for the specified paths and freshens the
|
|
directory cache.
|
|
|
|
If no paths are passed in then it will refresh the root directory.
|
|
|
|
rclone rc vfs/refresh
|
|
|
|
Otherwise pass directories in as dir=path. Any parameter key
|
|
starting with dir will refresh that directory, e.g.
|
|
|
|
rclone rc vfs/refresh dir=home/junk dir2=data/misc
|
|
|
|
If the parameter recursive=true is given the whole directory tree
|
|
will get refreshed. This refresh will use --fast-list if enabled.
|
|
` + getVFSHelp,
|
|
})
|
|
}
|
|
|
|
func rcRefresh(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
root, err := vfs.Root()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
getDir := func(path string) (*Dir, error) {
|
|
path = strings.Trim(path, "/")
|
|
segments := strings.Split(path, "/")
|
|
var node Node = root
|
|
for _, s := range segments {
|
|
if dir, ok := node.(*Dir); ok {
|
|
node, err = dir.stat(s)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
if dir, ok := node.(*Dir); ok {
|
|
return dir, nil
|
|
}
|
|
return nil, EINVAL
|
|
}
|
|
|
|
recursive := false
|
|
{
|
|
const k = "recursive"
|
|
|
|
if v, ok := in[k]; ok {
|
|
s, ok := v.(string)
|
|
if !ok {
|
|
return out, fmt.Errorf("value must be string %q=%v", k, v)
|
|
}
|
|
recursive, err = strconv.ParseBool(s)
|
|
if err != nil {
|
|
return out, fmt.Errorf("invalid value %q=%v", k, v)
|
|
}
|
|
delete(in, k)
|
|
}
|
|
}
|
|
|
|
result := map[string]string{}
|
|
if len(in) == 0 {
|
|
if recursive {
|
|
err = root.readDirTree()
|
|
} else {
|
|
err = root.readDir()
|
|
}
|
|
if err != nil {
|
|
result[""] = err.Error()
|
|
} else {
|
|
result[""] = "OK"
|
|
}
|
|
} else {
|
|
for k, v := range in {
|
|
path, ok := v.(string)
|
|
if !ok {
|
|
return out, fmt.Errorf("value must be string %q=%v", k, v)
|
|
}
|
|
if strings.HasPrefix(k, "dir") {
|
|
dir, err := getDir(path)
|
|
if err != nil {
|
|
result[path] = err.Error()
|
|
} else {
|
|
if recursive {
|
|
err = dir.readDirTree()
|
|
} else {
|
|
err = dir.readDir()
|
|
}
|
|
if err != nil {
|
|
result[path] = err.Error()
|
|
} else {
|
|
result[path] = "OK"
|
|
}
|
|
}
|
|
} else {
|
|
return out, fmt.Errorf("unknown key %q", k)
|
|
}
|
|
}
|
|
}
|
|
out = rc.Params{
|
|
"result": result,
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Add remote control for the VFS
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/forget",
|
|
Fn: rcForget,
|
|
Title: "Forget files or directories in the directory cache.",
|
|
Help: `
|
|
This forgets the paths in the directory cache causing them to be
|
|
re-read from the remote when needed.
|
|
|
|
If no paths are passed in then it will forget all the paths in the
|
|
directory cache.
|
|
|
|
rclone rc vfs/forget
|
|
|
|
Otherwise pass files or dirs in as file=path or dir=path. Any
|
|
parameter key starting with file will forget that file and any
|
|
starting with dir will forget that dir, e.g.
|
|
|
|
rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
|
|
` + getVFSHelp,
|
|
})
|
|
}
|
|
|
|
func rcForget(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
root, err := vfs.Root()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
forgotten := []string{}
|
|
if len(in) == 0 {
|
|
root.ForgetAll()
|
|
} else {
|
|
for k, v := range in {
|
|
path, ok := v.(string)
|
|
if !ok {
|
|
return out, fmt.Errorf("value must be string %q=%v", k, v)
|
|
}
|
|
path = strings.Trim(path, "/")
|
|
if strings.HasPrefix(k, "file") {
|
|
root.ForgetPath(path, fs.EntryObject)
|
|
} else if strings.HasPrefix(k, "dir") {
|
|
root.ForgetPath(path, fs.EntryDirectory)
|
|
} else {
|
|
return out, fmt.Errorf("unknown key %q", k)
|
|
}
|
|
forgotten = append(forgotten, path)
|
|
}
|
|
}
|
|
out = rc.Params{
|
|
"forgotten": forgotten,
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func getDuration(k string, v interface{}) (time.Duration, error) {
|
|
s, ok := v.(string)
|
|
if !ok {
|
|
return 0, fmt.Errorf("value must be string %q=%v", k, v)
|
|
}
|
|
interval, err := fs.ParseDuration(s)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("parse duration: %w", err)
|
|
}
|
|
return interval, nil
|
|
}
|
|
|
|
func getInterval(in rc.Params) (time.Duration, bool, error) {
|
|
k := "interval"
|
|
v, ok := in[k]
|
|
if !ok {
|
|
return 0, false, nil
|
|
}
|
|
interval, err := getDuration(k, v)
|
|
if err != nil {
|
|
return 0, true, err
|
|
}
|
|
if interval < 0 {
|
|
return 0, true, errors.New("interval must be >= 0")
|
|
}
|
|
delete(in, k)
|
|
return interval, true, nil
|
|
}
|
|
|
|
func getTimeout(in rc.Params) (time.Duration, error) {
|
|
k := "timeout"
|
|
v, ok := in[k]
|
|
if !ok {
|
|
return 10 * time.Second, nil
|
|
}
|
|
timeout, err := getDuration(k, v)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
delete(in, k)
|
|
return timeout, nil
|
|
}
|
|
|
|
func getStatus(vfs *VFS, in rc.Params) (out rc.Params, err error) {
|
|
for k, v := range in {
|
|
return nil, fmt.Errorf("invalid parameter: %s=%s", k, v)
|
|
}
|
|
return rc.Params{
|
|
"enabled": vfs.Opt.PollInterval != 0,
|
|
"supported": vfs.pollChan != nil,
|
|
"interval": map[string]interface{}{
|
|
"raw": vfs.Opt.PollInterval,
|
|
"seconds": time.Duration(vfs.Opt.PollInterval) / time.Second,
|
|
"string": vfs.Opt.PollInterval.String(),
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/poll-interval",
|
|
Fn: rcPollInterval,
|
|
Title: "Get the status or update the value of the poll-interval option.",
|
|
Help: `
|
|
Without any parameter given this returns the current status of the
|
|
poll-interval setting.
|
|
|
|
When the interval=duration parameter is set, the poll-interval value
|
|
is updated and the polling function is notified.
|
|
Setting interval=0 disables poll-interval.
|
|
|
|
rclone rc vfs/poll-interval interval=5m
|
|
|
|
The timeout=duration parameter can be used to specify a time to wait
|
|
for the current poll function to apply the new value.
|
|
If timeout is less or equal 0, which is the default, wait indefinitely.
|
|
|
|
The new poll-interval value will only be active when the timeout is
|
|
not reached.
|
|
|
|
If poll-interval is updated or disabled temporarily, some changes
|
|
might not get picked up by the polling function, depending on the
|
|
used remote.
|
|
` + getVFSHelp,
|
|
})
|
|
}
|
|
|
|
func rcPollInterval(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
interval, intervalPresent, err := getInterval(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
timeout, err := getTimeout(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for k, v := range in {
|
|
return nil, fmt.Errorf("invalid parameter: %s=%s", k, v)
|
|
}
|
|
if vfs.pollChan == nil {
|
|
return nil, errors.New("poll-interval is not supported by this remote")
|
|
}
|
|
|
|
if !intervalPresent {
|
|
return getStatus(vfs, in)
|
|
}
|
|
var timeoutHit bool
|
|
var timeoutChan <-chan time.Time
|
|
if timeout > 0 {
|
|
timer := time.NewTimer(timeout)
|
|
defer timer.Stop()
|
|
timeoutChan = timer.C
|
|
}
|
|
select {
|
|
case vfs.pollChan <- interval:
|
|
vfs.Opt.PollInterval = fs.Duration(interval)
|
|
case <-timeoutChan:
|
|
timeoutHit = true
|
|
}
|
|
out, err = getStatus(vfs, in)
|
|
if out != nil {
|
|
out["timeout"] = timeoutHit
|
|
}
|
|
return
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/list",
|
|
Title: "List active VFSes.",
|
|
Help: `
|
|
This lists the active VFSes.
|
|
|
|
It returns a list under the key "vfses" where the values are the VFS
|
|
names that could be passed to the other VFS commands in the "fs"
|
|
parameter.`,
|
|
Fn: rcList,
|
|
})
|
|
}
|
|
|
|
func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
activeMu.Lock()
|
|
defer activeMu.Unlock()
|
|
var names = []string{}
|
|
for name, vfses := range active {
|
|
if len(vfses) == 1 {
|
|
names = append(names, name)
|
|
} else {
|
|
for i := range vfses {
|
|
names = append(names, fmt.Sprintf("%s[%d]", name, i))
|
|
}
|
|
}
|
|
}
|
|
out = rc.Params{}
|
|
out["vfses"] = names
|
|
return out, nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/stats",
|
|
Title: "Stats for a VFS.",
|
|
Help: `
|
|
This returns stats for the selected VFS.
|
|
|
|
{
|
|
// Status of the disk cache - only present if --vfs-cache-mode > off
|
|
"diskCache": {
|
|
"bytesUsed": 0,
|
|
"erroredFiles": 0,
|
|
"files": 0,
|
|
"hashType": 1,
|
|
"outOfSpace": false,
|
|
"path": "/home/user/.cache/rclone/vfs/local/mnt/a",
|
|
"pathMeta": "/home/user/.cache/rclone/vfsMeta/local/mnt/a",
|
|
"uploadsInProgress": 0,
|
|
"uploadsQueued": 0
|
|
},
|
|
"fs": "/mnt/a",
|
|
"inUse": 1,
|
|
// Status of the in memory metadata cache
|
|
"metadataCache": {
|
|
"dirs": 1,
|
|
"files": 0
|
|
},
|
|
// Options as returned by options/get
|
|
"opt": {
|
|
"CacheMaxAge": 3600000000000,
|
|
// ...
|
|
"WriteWait": 1000000000
|
|
}
|
|
}
|
|
|
|
` + getVFSHelp,
|
|
Fn: rcStats,
|
|
})
|
|
}
|
|
|
|
func rcStats(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return vfs.Stats(), nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/queue",
|
|
Title: "Queue info for a VFS.",
|
|
Help: strings.ReplaceAll(`
|
|
This returns info about the upload queue for the selected VFS.
|
|
|
|
This is only useful if |--vfs-cache-mode| > off. If you call it when
|
|
the |--vfs-cache-mode| is off, it will return an empty result.
|
|
|
|
{
|
|
"queued": // an array of files queued for upload
|
|
[
|
|
{
|
|
"name": "file", // string: name (full path) of the file,
|
|
"id": 123, // integer: id of this item in the queue,
|
|
"size": 79, // integer: size of the file in bytes
|
|
"expiry": 1.5 // float: time until file is eligible for transfer, lowest goes first
|
|
"tries": 1, // integer: number of times we have tried to upload
|
|
"delay": 5.0, // float: seconds between upload attempts
|
|
"uploading": false, // boolean: true if item is being uploaded
|
|
},
|
|
],
|
|
}
|
|
|
|
The |expiry| time is the time until the file is elegible for being
|
|
uploaded in floating point seconds. This may go negative. As rclone
|
|
only transfers |--transfers| files at once, only the lowest
|
|
|--transfers| expiry times will have |uploading| as |true|. So there
|
|
may be files with negative expiry times for which |uploading| is
|
|
|false|.
|
|
|
|
`, "|", "`") + getVFSHelp,
|
|
Fn: rcQueue,
|
|
})
|
|
}
|
|
|
|
func rcQueue(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if vfs.cache == nil {
|
|
return nil, nil
|
|
}
|
|
return vfs.cache.Queue(), nil
|
|
}
|
|
|
|
func init() {
|
|
rc.Add(rc.Call{
|
|
Path: "vfs/queue-set-expiry",
|
|
Title: "Set the expiry time for an item queued for upload.",
|
|
Help: strings.ReplaceAll(`
|
|
|
|
Use this to adjust the |expiry| time for an item in the upload queue.
|
|
You will need to read the |id| of the item using |vfs/queue| before
|
|
using this call.
|
|
|
|
You can then set |expiry| to a floating point number of seconds from
|
|
now when the item is eligible for upload. If you want the item to be
|
|
uploaded as soon as possible then set it to a large negative number (eg
|
|
-1000000000). If you want the upload of the item to be delayed
|
|
for a long time then set it to a large positive number.
|
|
|
|
Setting the |expiry| of an item which has already has started uploading
|
|
will have no effect - the item will carry on being uploaded.
|
|
|
|
This will return an error if called with |--vfs-cache-mode| off or if
|
|
the |id| passed is not found.
|
|
|
|
This takes the following parameters
|
|
|
|
- |fs| - select the VFS in use (optional)
|
|
- |id| - a numeric ID as returned from |vfs/queue|
|
|
- |expiry| - a new expiry time as floating point seconds
|
|
|
|
This returns an empty result on success, or an error.
|
|
|
|
`, "|", "`") + getVFSHelp,
|
|
Fn: rcQueueSetExpiry,
|
|
})
|
|
}
|
|
|
|
func rcQueueSetExpiry(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
|
vfs, err := getVFS(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if vfs.cache == nil {
|
|
return nil, rc.NewErrParamInvalid(errors.New("can't call this unless using the VFS cache"))
|
|
}
|
|
|
|
// Read input values
|
|
id, err := in.GetInt64("id")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
expiry, err := in.GetFloat64("expiry")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set expiry
|
|
expiryTime := time.Now().Add(time.Duration(float64(time.Second) * expiry))
|
|
err = vfs.cache.QueueSetExpiry(writeback.Handle(id), expiryTime)
|
|
return nil, err
|
|
}
|