From 49d7b0d27823e39fb41ba865344fb88f1e2fa6bc Mon Sep 17 00:00:00 2001 From: Gary Kim Date: Thu, 25 Apr 2019 09:51:15 +0000 Subject: [PATCH] sftp: add About support - fixes #3107 This adds support for using About with SFTP remotes. This works by calling the df command remotely. --- backend/sftp/sftp.go | 70 ++++++++++++++++++++++++++++++ backend/sftp/sftp_internal_test.go | 14 ++++++ docs/content/overview.md | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index c42e94057..ddec47d5d 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -14,6 +14,7 @@ import ( "os/user" "path" "regexp" + "strconv" "strings" "sync" "time" @@ -813,6 +814,47 @@ func (f *Fs) Hashes() hash.Set { return set } +// About gets usage stats +func (f *Fs) About() (*fs.Usage, error) { + c, err := f.getSftpConnection() + if err != nil { + return nil, errors.Wrap(err, "About get SFTP connection") + } + session, err := c.sshClient.NewSession() + f.putSftpConnection(&c, err) + if err != nil { + return nil, errors.Wrap(err, "About put SFTP connection") + } + + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + escapedPath := shellEscape(f.root) + if f.opt.PathOverride != "" { + escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root)) + } + if len(escapedPath) == 0 { + escapedPath = "/" + } + err = session.Run("df -k " + escapedPath) + if err != nil { + _ = session.Close() + return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.") + } + _ = session.Close() + + usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes()) + if usageTotal < 0 || usageUsed < 0 || usageAvail < 0 { + return nil, errors.Wrap(err, "About failed to parse information") + } + usage := &fs.Usage{ + Total: fs.NewUsageValue(usageTotal), + Used: fs.NewUsageValue(usageUsed), + Free: fs.NewUsageValue(usageAvail), + } + return usage, nil +} + // Fs is the filesystem this remote sftp file object is located within func (o *Object) Fs() fs.Info { return o.fs @@ -903,6 +945,34 @@ func parseHash(bytes []byte) string { return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator } +// Parses the byte array output from the SSH session +// returned by an invocation of df into +// the disk size, used space, and avaliable space on the disk, in that order. +// Only works when `df` has output info on only one disk +func parseUsage(bytes []byte) (int64, int64, int64) { + lines := strings.Split(string(bytes), "\n") + if len(lines) < 2 { + return -1, -1, -1 + } + split := strings.Fields(lines[1]) + if len(split) < 6 { + return -1, -1, -1 + } + spaceTotal, err := strconv.ParseInt(split[1], 10, 64) + if err != nil { + return -1, -1, -1 + } + spaceUsed, err := strconv.ParseInt(split[2], 10, 64) + if err != nil { + return -1, -1, -1 + } + spaceAvail, err := strconv.ParseInt(split[3], 10, 64) + if err != nil { + return -1, -1, -1 + } + return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024 +} + // Size returns the size in bytes of the remote sftp file func (o *Object) Size() int64 { return o.size diff --git a/backend/sftp/sftp_internal_test.go b/backend/sftp/sftp_internal_test.go index 5d2741885..e23ae8db9 100644 --- a/backend/sftp/sftp_internal_test.go +++ b/backend/sftp/sftp_internal_test.go @@ -35,3 +35,17 @@ func TestParseHash(t *testing.T) { assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput)) } } + +func TestParseUsage(t *testing.T) { + for i, test := range []struct { + sshOutput string + usage [3]int64 + }{ + {"Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/root 91283092 81111888 10154820 89% /", [3]int64{93473886208, 83058573312, 10398535680}}, + {"Filesystem 1K-blocks Used Available Use% Mounted on\ntmpfs 818256 1636 816620 1% /run", [3]int64{837894144, 1675264, 836218880}}, + {"Filesystem 1024-blocks Used Available Capacity iused ifree %iused Mounted on\n/dev/disk0s2 244277768 94454848 149566920 39% 997820 4293969459 0% /", [3]int64{250140434432, 96721764352, 153156526080}}, + } { + gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput)) + assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput)) + } +} diff --git a/docs/content/overview.md b/docs/content/overview.md index 87e020e25..d79fa6b23 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -149,7 +149,7 @@ operations more efficient. | Openstack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | | QingStor | No | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | -| SFTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No | +| SFTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | The local filesystem | Yes | No | Yes | Yes | No | No | Yes | No | Yes |