Merge pull request #1657 from restic/rclone-backend

Rclone backend

Gopkg.lock

@@ -232,6 +232,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
+  inputs-digest = "d3d59414a33bb8ecc6d88a681c782a87244a565cc9d0f85615cfa0704c02800a"
   solver-name = "gps-cdcl"
   solver-version = 1

README.rst

@@ -57,6 +57,7 @@ Therefore, restic supports the following backends for storing backups natively:
 - `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
 - `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
 - `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
+- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
 
 Design Principles
 -----------------

changelog/unreleased/issue-1561

@@ -0,0 +1,10 @@
+Enhancement: Allow using rclone to access other services
+
+We've added the ability to use rclone to store backup data on all backends that
+it supports. This was done in collaboration with Nick, the author of rclone.
+You can now use it to first configure a service, then restic manages the rest
+(starting and stopping rclone). For details, please see the manual.
+
+https://github.com/restic/restic/issues/1561
+https://github.com/restic/restic/pull/1657
+https://rclone.org

cmd/restic/global.go

@@ -18,6 +18,7 @@ import (
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			return nil, err
 		}
 
+		debug.Log("opening rest repository at %#v", cfg)
+		return cfg, nil
+	case "rclone":
+		cfg := loc.Config.(rclone.Config)
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
 		debug.Log("opening rest repository at %#v", cfg)
 		return cfg, nil
 	}
@@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config), rt)
+	case "rclone":
+		be, err = rclone.Open(cfg.(rclone.Config))
 
 	default:
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		return rest.Create(cfg.(rest.Config), rt)
+	case "rclone":
+		return rclone.Open(cfg.(rclone.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

doc/030_preparing_a_new_repo.rst

@@ -139,7 +139,7 @@ If you use TLS, restic will use the system's CA certificates to verify the
 server certificate. When the verification fails, restic refuses to proceed and
 exits with an error. If you have your own self-signed certificate, or a custom
 CA certificate should be used for verification, you can pass restic the
-certificate filename via the `--cacert` option.
+certificate filename via the ``--cacert`` option.
 
 REST server uses exactly the same directory structure as local backend,
 so you should be able to access it both locally and via HTTP, even
@@ -306,8 +306,8 @@ bucket does not exist yet, it will be created:
     Please note that knowledge of your password is required to access the repository.
     Losing your password means that your data is irrecoverably lost.
 
-The number of concurrent connections to the B2 service can be set with the `-o
+The number of concurrent connections to the B2 service can be set with the ``-o
-b2.connections=10`. By default, at most five parallel connections are
+b2.connections=10``. By default, at most five parallel connections are
 established.
 
 Microsoft Azure Blob Storage
@@ -321,7 +321,7 @@ account name and key as follows:
     $ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
     $ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
 
-Afterwards you can initialize a repository in a container called `foo` in the
+Afterwards you can initialize a repository in a container called ``foo`` in the
 root path like this:
 
 .. code-block:: console
@@ -334,7 +334,7 @@ root path like this:
     [...]
 
 The number of concurrent connections to the Azure Blob Storage service can be set with the
-`-o azure.connections=10`. By default, at most five parallel connections are
+``-o azure.connections=10``. By default, at most five parallel connections are
 established.
 
 Google Cloud Storage
@@ -369,7 +369,7 @@ located on an instance with default service accounts then these should work out
 the box.
 
 Once authenticated, you can use the ``gs:`` backend type to create a new
-repository in the bucket `foo` at the root path:
+repository in the bucket ``foo`` at the root path:
 
 .. code-block:: console
 
@@ -381,12 +381,117 @@ repository in the bucket `foo` at the root path:
     [...]
 
 The number of concurrent connections to the GCS service can be set with the
-`-o gs.connections=10`. By default, at most five parallel connections are
+``-o gs.connections=10``. By default, at most five parallel connections are
 established.
 
 .. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
 .. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
 
+Other Services via rclone
+*************************
+
+The program `rclone`_ can be used to access many other different services and
+store data there. First, you need to install and `configure`_ rclone.  The
+general backend specification format is ``rclone:<remote>:<path>``, the
+``<remote>:<path>`` component will be directly passed to rclone. When you
+configure a remote named ``foo``, you can then call restic as follows to
+initiate a new repository in the path ``bar`` in the repo:
+
+.. code-block:: console
+
+    $ restic -r rclone:foo:bar init
+
+Restic takes care of starting and stopping rclone.
+
+As a more concrete example, suppose you have configured a remote named
+``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
+You can then use rclone to list files in the bucket like this:
+
+.. code-block:: console
+
+    $ rclone ls b2prod:yggdrasil
+
+In order to create a new repository in the root directory of the bucket, call
+restic like this:
+
+.. code-block:: console
+
+    $ restic -r rclone:b2prod:yggdrasil init
+
+If you want to use the path ``foo/bar/baz`` in the bucket instead, pass this to
+restic:
+
+.. code-block:: console
+
+    $ restic -r rclone:b2prod:yggdrasil/foo/bar/baz init
+
+Listing the files of an empty repository directly with rclone should return a
+listing similar to the following:
+
+.. code-block:: console
+
+    $ rclone ls b2prod:yggdrasil/foo/bar/baz
+        155 bar/baz/config
+        448 bar/baz/keys/4bf9c78049de689d73a56ed0546f83b8416795295cda12ec7fb9465af3900b44
+
+Rclone can be `configured with environment variables`_, so for instance
+configuring a bandwidth limit for rclone cat be achieve by setting the
+``RCLONE_BWLIMIT`` environment variable:
+
+.. code-block:: console
+
+    $ export RCLONE_BWLIMIT=1M
+
+For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
+
+The rclone backend has two additional options:
+
+ * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
+ * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete --drive-use-trash=false``
+
+The reason why the two last parameters (``--b2-hard-delete`` and
+``--drive-use-trash=false``) can be found in the corresponding GitHub `issue #1657`_.
+
+In order to start rclone, restic will build a list of arguments by joining the
+following lists (in this order): ``rclone.program``, ``rclone.args`` and as the
+last parameter the value that follows the ``rclone:`` prefix of the repository
+specification.
+
+So, calling restic like this
+
+.. code-block:: console
+
+    $ restic -o rclone.program="/path/to/rclone" \
+      -o rclone.args="serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose" \
+      -r rclone:b2:foo/bar
+
+runs rclone as follows:
+
+.. code-block:: console
+
+    $ /path/to/rclone serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose b2:foo/bar
+
+Manually setting ``rclone.program`` also allows running a remote instance of
+rclone e.g. via SSH on a server, for example:
+
+.. code-block:: console
+
+    $ restic -o rclone.program="ssh user@host rclone" -r rclone:b2:foo/bar
+
+The rclone command may also be hard-coded in the SSH configuration or the
+user's public key, in this case it may be sufficient to just start the SSH
+connection (and it's irrelevant what's passed after ``rclone:`` in the
+repository specification):
+
+.. code-block:: console
+
+    $ restic -o rclone.program="ssh user@host" -r rclone:x
+
+.. _rclone: https://rclone.org/
+.. _configure: https://rclone.org/docs/
+.. _configured with environment variables: https://rclone.org/docs/#environment-variables
+.. _issue #1657: https://github.com/restic/restic/pull/1657#issuecomment-377707486
+
 Password prompt on Windows
 **************************
 

doc/100_references.rst

@@ -672,8 +672,8 @@ The following values are valid for ``{type}``:
 The API version is selected via the ``Accept`` HTTP header in the request. The
 following values are defined:
 
- * ``application/vnd.x.restic.rest.v1+json`` or empty: Select API version 1
+ * ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1
- * ``application/vnd.x.restic.rest.v2+json``: Select API version 2
+ * ``application/vnd.x.restic.rest.v2``: Select API version 2
 
 The server will respond with the value of the highest version it supports in
 the ``Content-Type`` HTTP response header for the HTTP requests which should
@@ -681,7 +681,7 @@ return JSON. Any different value for this header means API version 1.
 
 The placeholder ``{path}`` in this document is a path to the repository, so
 that multiple different repositories can be accessed. The default path is
-``/``.
+``/``. The path must end with a slash.
 
 POST {path}?create=true
 =======================

internal/backend/foreground_solaris.go

@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"os/exec"
@@ -7,7 +7,10 @@ import (
 	"github.com/restic/restic/internal/errors"
 )
 
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+// StartForeground runs cmd in the foreground, by temporarily switching to the
+// new process group created for cmd. The returned function `bg` switches back
+// to the previous process group.
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// run the command in it's own process group so that SIGINT
 	// is not sent to it.
 	cmd.SysProcAttr = &syscall.SysProcAttr{

internal/backend/foreground_unix.go

@@ -1,7 +1,7 @@
 // +build !solaris
 // +build !windows
 
-package sftp
+package backend
 
 import (
 	"os"
@@ -24,10 +24,10 @@ func tcsetpgrp(fd int, pid int) error {
 	return errno
 }
 
-// startForeground runs cmd in the foreground, by temporarily switching to the
+// StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// open the TTY, we need the file descriptor
 	tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
 	if err != nil {

internal/backend/foreground_windows.go

@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"os/exec"
@@ -6,10 +6,10 @@ import (
 	"github.com/restic/restic/internal/errors"
 )
 
-// startForeground runs cmd in the foreground, by temporarily switching to the
+// StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// just start the process and hope for the best
 	err = cmd.Start()
 	if err != nil {

internal/backend/location/location.go

@@ -8,6 +8,7 @@ import (
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -38,6 +39,7 @@ var parsers = []parser{
 	{"azure", azure.ParseConfig},
 	{"swift", swift.ParseConfig},
 	{"rest", rest.ParseConfig},
+	{"rclone", rclone.ParseConfig},
 }
 
 func isPath(s string) bool {

internal/backend/rclone/backend.go

@@ -0,0 +1,280 @@
+package rclone
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"sync"
+	"time"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"golang.org/x/net/context/ctxhttp"
+	"golang.org/x/net/http2"
+)
+
+// Backend is used to access data stored somewhere via rclone.
+type Backend struct {
+	*rest.Backend
+	tr         *http2.Transport
+	cmd        *exec.Cmd
+	waitCh     <-chan struct{}
+	waitResult error
+	wg         *sync.WaitGroup
+	conn       *StdioConn
+}
+
+// run starts command with args and initializes the StdioConn.
+func run(command string, args ...string) (*StdioConn, *exec.Cmd, *sync.WaitGroup, func() error, error) {
+	cmd := exec.Command(command, args...)
+
+	p, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	var wg sync.WaitGroup
+
+	// start goroutine to add a prefix to all messages printed by to stderr by rclone
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		sc := bufio.NewScanner(p)
+		for sc.Scan() {
+			fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
+		}
+	}()
+
+	r, stdin, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	stdout, w, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	cmd.Stdin = r
+	cmd.Stdout = w
+
+	bg, err := backend.StartForeground(cmd)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	c := &StdioConn{
+		stdin:  stdout,
+		stdout: stdin,
+		cmd:    cmd,
+	}
+
+	return c, cmd, &wg, bg, nil
+}
+
+// New initializes a Backend and starts the process.
+func New(cfg Config) (*Backend, error) {
+	var (
+		args []string
+		err  error
+	)
+
+	// build program args, start with the program
+	if cfg.Program != "" {
+		a, err := backend.SplitShellStrings(cfg.Program)
+		if err != nil {
+			return nil, err
+		}
+		args = append(args, a...)
+	} else {
+		args = append(args, "rclone")
+	}
+
+	// then add the arguments
+	if cfg.Args != "" {
+		a, err := backend.SplitShellStrings(cfg.Args)
+		if err != nil {
+			return nil, err
+		}
+
+		args = append(args, a...)
+	} else {
+		args = append(args,
+			"serve", "restic", "--stdio",
+			"--b2-hard-delete", "--drive-use-trash=false")
+	}
+
+	// finally, add the remote
+	args = append(args, cfg.Remote)
+	arg0, args := args[0], args[1:]
+
+	debug.Log("running command: %v %v", arg0, args)
+	conn, cmd, wg, bg, err := run(arg0, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	dialCount := 0
+	tr := &http2.Transport{
+		AllowHTTP: true, // this is not really HTTP, just stdin/stdout
+		DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
+			debug.Log("new connection requested, %v %v", network, address)
+			if dialCount > 0 {
+				panic("dial count > 0")
+			}
+			dialCount++
+			return conn, nil
+		},
+	}
+
+	waitCh := make(chan struct{})
+	be := &Backend{
+		tr:     tr,
+		cmd:    cmd,
+		waitCh: waitCh,
+		conn:   conn,
+		wg:     wg,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		debug.Log("waiting for error result")
+		err := cmd.Wait()
+		debug.Log("Wait returned %v", err)
+		be.waitResult = err
+		close(waitCh)
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		debug.Log("monitoring command to cancel first HTTP request context")
+		select {
+		case <-ctx.Done():
+			debug.Log("context has been cancelled, returning")
+		case <-be.waitCh:
+			debug.Log("command has exited, cancelling context")
+			cancel()
+		}
+	}()
+
+	// send an HTTP request to the base URL, see if the server is there
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   60 * time.Second,
+	}
+
+	// request a random file which does not exist. we just want to test when
+	// rclone is able to accept HTTP requests.
+	url := fmt.Sprintf("http://localhost/file-%d", rand.Uint64())
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", rest.ContentTypeV2)
+	req.Cancel = ctx.Done()
+
+	res, err := ctxhttp.Do(ctx, client, req)
+	if err != nil {
+		bg()
+		_ = cmd.Process.Kill()
+		return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
+	}
+
+	debug.Log("HTTP status %q returned, moving instance to background", res.Status)
+	bg()
+
+	return be, nil
+}
+
+// Open starts an rclone process with the given config.
+func Open(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: cfg.Connections,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Open(restConfig, be.tr)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+// Create initializes a new restic repo with clone.
+func Create(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("new backend created")
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: 20,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Create(restConfig, be.tr)
+	if err != nil {
+		_ = be.Close()
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+const waitForExit = 5 * time.Second
+
+// Close terminates the backend.
+func (be *Backend) Close() error {
+	debug.Log("exiting rclone")
+	be.tr.CloseIdleConnections()
+
+	select {
+	case <-be.waitCh:
+		debug.Log("rclone exited")
+	case <-time.After(waitForExit):
+		debug.Log("timeout, closing file descriptors")
+		err := be.conn.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	be.wg.Wait()
+	debug.Log("wait for rclone returned: %v", be.waitResult)
+	return be.waitResult
+}

internal/backend/rclone/backend_test.go

@@ -0,0 +1,66 @@
+package rclone_test
+
+import (
+	"os/exec"
+	"testing"
+
+	"github.com/restic/restic/internal/backend/rclone"
+	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func newTestSuite(t testing.TB) *test.Suite {
+	dir, cleanup := rtest.TempDir(t)
+
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			t.Logf("use backend at %v", dir)
+			cfg := rclone.NewConfig()
+			cfg.Remote = dir
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Create()")
+			cfg := config.(rclone.Config)
+			be, err := rclone.Create(cfg)
+			if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound {
+				t.Skipf("program %q not found", e.Name)
+				return nil, nil
+			}
+			return be, err
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Open()")
+			cfg := config.(rclone.Config)
+			return rclone.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			t.Logf("cleanup dir %v", dir)
+			cleanup()
+			return nil
+		},
+	}
+}
+
+func TestBackendRclone(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")
+		}
+	}()
+
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendREST(t *testing.B) {
+	newTestSuite(t).RunBenchmarks(t)
+}

internal/backend/rclone/config.go

@@ -0,0 +1,39 @@
+package rclone
+
+import (
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/options"
+)
+
+// Config contains all configuration necessary to start rclone.
+type Config struct {
+	Program     string `option:"program" help:"path to rclone (default: rclone)"`
+	Args        string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio --b2-hard-delete --drive-use-trash=false)"`
+	Remote      string
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+}
+
+func init() {
+	options.Register("rclone", Config{})
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+// ParseConfig parses the string s and extracts the remote server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rclone:") {
+		return nil, errors.New("invalid rclone backend specification")
+	}
+
+	s = s[7:]
+	cfg := NewConfig()
+	cfg.Remote = s
+	return cfg, nil
+}

internal/backend/rclone/config_test.go

@@ -0,0 +1,34 @@
+package rclone
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	var tests = []struct {
+		s   string
+		cfg Config
+	}{
+		{
+			"rclone:local:foo:/bar",
+			Config{
+				Remote:      "local:foo:/bar",
+				Connections: 5,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(cfg, test.cfg) {
+				t.Fatalf("wrong config, want:\n  %v\ngot:\n  %v", test.cfg, cfg)
+			}
+		})
+	}
+}

internal/backend/rclone/stdio_conn.go

@@ -0,0 +1,74 @@
+package rclone
+
+import (
+	"net"
+	"os"
+	"os/exec"
+	"sync"
+
+	"github.com/restic/restic/internal/debug"
+)
+
+// StdioConn implements a net.Conn via stdin/stdout.
+type StdioConn struct {
+	stdin  *os.File
+	stdout *os.File
+	cmd    *exec.Cmd
+	close  sync.Once
+}
+
+func (s *StdioConn) Read(p []byte) (int, error) {
+	n, err := s.stdin.Read(p)
+	return n, err
+}
+
+func (s *StdioConn) Write(p []byte) (int, error) {
+	n, err := s.stdout.Write(p)
+	return n, err
+}
+
+// Close closes both streams.
+func (s *StdioConn) Close() (err error) {
+	s.close.Do(func() {
+		debug.Log("close stdio connection")
+		var errs []error
+
+		for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
+			err := f()
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+
+		if len(errs) > 0 {
+			err = errs[0]
+		}
+	})
+
+	return err
+}
+
+// LocalAddr returns nil.
+func (s *StdioConn) LocalAddr() net.Addr {
+	return Addr{}
+}
+
+// RemoteAddr returns nil.
+func (s *StdioConn) RemoteAddr() net.Addr {
+	return Addr{}
+}
+
+// make sure StdioConn implements net.Conn
+var _ net.Conn = &StdioConn{}
+
+// Addr implements net.Addr for stdin/stdout.
+type Addr struct{}
+
+// Network returns the network type as a string.
+func (a Addr) Network() string {
+	return "stdio"
+}
+
+func (a Addr) String() string {
+	return "stdio"
+}

internal/backend/rclone/stdio_conn_go110.go

@@ -0,0 +1,25 @@
+// +build go1.10
+
+package rclone
+
+import "time"
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	err1 := s.stdin.SetReadDeadline(t)
+	err2 := s.stdout.SetWriteDeadline(t)
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return s.stdin.SetReadDeadline(t)
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return s.stdout.SetWriteDeadline(t)
+}

internal/backend/rclone/stdio_conn_other.go

@@ -0,0 +1,22 @@
+// +build !go1.10
+
+package rclone
+
+import "time"
+
+// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that.
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

internal/backend/rest/config.go

@@ -32,6 +32,10 @@ func ParseConfig(s string) (interface{}, error) {
 	}
 
 	s = s[5:]
+	if !strings.HasSuffix(s, "/") {
+		s += "/"
+	}
+
 	u, err := url.Parse(s)
 
 	if err != nil {

internal/backend/rest/config_test.go

@@ -19,24 +19,34 @@ var configTests = []struct {
 	s   string
 	cfg Config
 }{
-	{"rest:http://localhost:1234", Config{
+	{
-		URL:         parseURL("http://localhost:1234"),
+		s: "rest:http://localhost:1234",
+		cfg: Config{
+			URL:         parseURL("http://localhost:1234/"),
 			Connections: 5,
-	}},
+		},
+	},
+	{
+		s: "rest:http://localhost:1234/",
+		cfg: Config{
+			URL:         parseURL("http://localhost:1234/"),
+			Connections: 5,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {
-	for i, test := range configTests {
+	for _, test := range configTests {
+		t.Run("", func(t *testing.T) {
 			cfg, err := ParseConfig(test.s)
 			if err != nil {
-			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+				t.Fatalf("%s failed: %v", test.s, err)
-			continue
 			}
 
 			if !reflect.DeepEqual(cfg, test.cfg) {
-			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				t.Fatalf("\ninput: %s\n wrong config, want:\n  %v\ngot:\n  %v",
-				i, test.s, test.cfg, cfg)
+					test.s, test.cfg, cfg)
-			continue
 			}
+		})
 	}
 }

internal/backend/rest/rest.go

@@ -21,22 +21,24 @@ import (
 )
 
 // make sure the rest backend implements restic.Backend
-var _ restic.Backend = &restBackend{}
+var _ restic.Backend = &Backend{}
 
-type restBackend struct {
+// Backend uses the REST protocol to access data stored on a server.
+type Backend struct {
 	url    *url.URL
 	sem    *backend.Semaphore
 	client *http.Client
 	backend.Layout
 }
 
+// the REST API protocol version is decided by HTTP request headers, these are the constants.
 const (
-	contentTypeV1 = "application/vnd.x.restic.rest.v1"
+	ContentTypeV1 = "application/vnd.x.restic.rest.v1"
-	contentTypeV2 = "application/vnd.x.restic.rest.v2"
+	ContentTypeV2 = "application/vnd.x.restic.rest.v2"
 )
 
 // Open opens the REST backend with the given config.
-func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
+func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	client := &http.Client{Transport: rt}
 
 	sem, err := backend.NewSemaphore(cfg.Connections)
@@ -50,7 +52,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
 		url = url[:len(url)-1]
 	}
 
-	be := &restBackend{
+	be := &Backend{
 		url:    cfg.URL,
 		client: client,
 		Layout: &backend.RESTLayout{URL: url, Join: path.Join},
@@ -61,7 +63,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
 }
 
 // Create creates a new REST on server configured in config.
-func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	be, err := Open(cfg, rt)
 	if err != nil {
 		return nil, err
@@ -100,12 +102,12 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
 }
 
 // Location returns this backend's location (the server's URL).
-func (b *restBackend) Location() string {
+func (b *Backend) Location() string {
 	return b.url.String()
 }
 
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -119,7 +121,7 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
 		return errors.Wrap(err, "NewRequest")
 	}
 	req.Header.Set("Content-Type", "application/octet-stream")
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	// explicitly set the content length, this prevents chunked encoding and
 	// let's the server know what's coming.
@@ -162,7 +164,7 @@ func (e ErrIsNotExist) Error() string {
 }
 
 // IsNotExist returns true if the error was caused by a non-existing file.
-func (b *restBackend) IsNotExist(err error) bool {
+func (b *Backend) IsNotExist(err error) bool {
 	err = errors.Cause(err)
 	_, ok := err.(ErrIsNotExist)
 	return ok
@@ -170,11 +172,11 @@ func (b *restBackend) IsNotExist(err error) bool {
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
 }
 
-func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
 	debug.Log("Load %v, length %v, offset %v", h, length, offset)
 	if err := h.Valid(); err != nil {
 		return nil, err
@@ -198,7 +200,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
 		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
 	}
 	req.Header.Set("Range", byteRange)
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 	debug.Log("Load(%v) send range %v", h, byteRange)
 
 	b.sem.GetToken()
@@ -227,7 +229,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
 }
 
 // Stat returns information about a blob.
-func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -236,7 +238,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -272,7 +274,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
 	_, err := b.Stat(ctx, h)
 	if err != nil {
 		return false, nil
@@ -282,7 +284,7 @@ func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
+func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -291,7 +293,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 	if err != nil {
 		return errors.Wrap(err, "http.NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -320,7 +322,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
 	url := b.Dirname(restic.Handle{Type: t})
 	if !strings.HasSuffix(url, "/") {
 		url += "/"
@@ -330,7 +332,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 	if err != nil {
 		return errors.Wrap(err, "NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -344,7 +346,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 		return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode)
 	}
 
-	if resp.Header.Get("Content-Type") == contentTypeV2 {
+	if resp.Header.Get("Content-Type") == ContentTypeV2 {
 		return b.listv2(ctx, t, resp, fn)
 	}
 
@@ -354,7 +356,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
 // /data/`) only returns the names of the files, so we need to issue an HTTP
 // HEAD request for each file.
-func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
 	debug.Log("parsing API v1 response")
 	dec := json.NewDecoder(resp.Body)
 	var list []string
@@ -388,7 +390,7 @@ func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.
 
 // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
 // /data/`) returns the names and sizes of all files.
-func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+func (b *Backend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
 	debug.Log("parsing API v2 response")
 	dec := json.NewDecoder(resp.Body)
 
@@ -424,21 +426,21 @@ func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.
 }
 
 // Close closes all open files.
-func (b *restBackend) Close() error {
+func (b *Backend) Close() error {
 	// this does not need to do anything, all open files are closed within the
 	// same function.
 	return nil
 }
 
 // Remove keys for a specified backend type.
-func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error {
+func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
 	return b.List(ctx, t, func(fi restic.FileInfo) error {
 		return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
 	})
 }
 
 // Delete removes all data in the backend.
-func (b *restBackend) Delete(ctx context.Context) error {
+func (b *Backend) Delete(ctx context.Context) error {
 	alltypes := []restic.FileType{
 		restic.DataFile,
 		restic.KeyFile,

internal/backend/sftp/sftp.go

@@ -65,7 +65,7 @@ func startClient(program string, args ...string) (*SFTP, error) {
 		return nil, errors.Wrap(err, "cmd.StdoutPipe")
 	}
 
-	bg, err := startForeground(cmd)
+	bg, err := backend.StartForeground(cmd)
 	if err != nil {
 		return nil, errors.Wrap(err, "cmd.Start")
 	}
@@ -179,7 +179,12 @@ func (r *SFTP) IsNotExist(err error) bool {
 
 func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
 	if cfg.Command != "" {
-		return SplitShellArgs(cfg.Command)
+		args, err := backend.SplitShellStrings(cfg.Command)
+		if err != nil {
+			return "", nil, err
+		}
+
+		return args[0], args[1:], nil
 	}
 
 	cmd = "ssh"

internal/backend/shell_split.go

@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"unicode"
@@ -41,8 +41,8 @@ func (s *shellSplitter) isSplitChar(c rune) bool {
 	return c == '\\' || unicode.IsSpace(c)
 }
 
-// SplitShellArgs returns the list of arguments from a shell command string.
+// SplitShellStrings returns the list of shell strings from a shell command string.
-func SplitShellArgs(data string) (cmd string, args []string, err error) {
+func SplitShellStrings(data string) (strs []string, err error) {
 	s := &shellSplitter{}
 
 	// derived from strings.SplitFunc
@@ -50,7 +50,7 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
 	for i, rune := range data {
 		if s.isSplitChar(rune) {
 			if fieldStart >= 0 {
-				args = append(args, data[fieldStart:i])
+				strs = append(strs, data[fieldStart:i])
 				fieldStart = -1
 			}
 		} else if fieldStart == -1 {
@@ -58,21 +58,19 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
 		}
 	}
 	if fieldStart >= 0 { // Last field might end at EOF.
-		args = append(args, data[fieldStart:])
+		strs = append(strs, data[fieldStart:])
 	}
 
 	switch s.quote {
 	case '\'':
-		return "", nil, errors.New("single-quoted string not terminated")
+		return nil, errors.New("single-quoted string not terminated")
 	case '"':
-		return "", nil, errors.New("double-quoted string not terminated")
+		return nil, errors.New("double-quoted string not terminated")
 	}
 
-	if len(args) == 0 {
+	if len(strs) == 0 {
-		return "", nil, errors.New("command string is empty")
+		return nil, errors.New("command string is empty")
 	}
 
-	cmd, args = args[0], args[1:]
+	return strs, nil
-
-	return cmd, args, nil
 }

internal/backend/shell_split_test.go

@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"reflect"
@@ -8,59 +8,53 @@ import (
 func TestShellSplitter(t *testing.T) {
 	var tests = []struct {
 		data string
-		cmd  string
 		args []string
 	}{
 		{
 			`foo`,
-			"foo", []string{},
+			[]string{"foo"},
 		},
 		{
 			`'foo'`,
-			"foo", []string{},
+			[]string{"foo"},
 		},
 		{
 			`foo bar baz`,
-			"foo", []string{"bar", "baz"},
+			[]string{"foo", "bar", "baz"},
 		},
 		{
 			`foo 'bar' baz`,
-			"foo", []string{"bar", "baz"},
+			[]string{"foo", "bar", "baz"},
 		},
 		{
 			`'bar box' baz`,
-			"bar box", []string{"baz"},
+			[]string{"bar box", "baz"},
 		},
 		{
 			`"bar 'box'" baz`,
-			"bar 'box'", []string{"baz"},
+			[]string{"bar 'box'", "baz"},
 		},
 		{
 			`'bar "box"' baz`,
-			`bar "box"`, []string{"baz"},
+			[]string{`bar "box"`, "baz"},
 		},
 		{
 			`\"bar box baz`,
-			`"bar`, []string{"box", "baz"},
+			[]string{`"bar`, "box", "baz"},
 		},
 		{
 			`"bar/foo/x" "box baz"`,
-			"bar/foo/x", []string{"box baz"},
+			[]string{"bar/foo/x", "box baz"},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			cmd, args, err := SplitShellArgs(test.data)
+			args, err := SplitShellStrings(test.data)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			if cmd != test.cmd {
-				t.Fatalf("wrong cmd returned, want:\n  %#v\ngot:\n  %#v",
-					test.cmd, cmd)
-			}
-
 			if !reflect.DeepEqual(args, test.args) {
 				t.Fatalf("wrong args returned, want:\n  %#v\ngot:\n  %#v",
 					test.args, args)
@@ -94,7 +88,7 @@ func TestShellSplitterInvalid(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			cmd, args, err := SplitShellArgs(test.data)
+			args, err := SplitShellStrings(test.data)
 			if err == nil {
 				t.Fatalf("expected error not found: %v", test.err)
 			}
@@ -103,10 +97,6 @@ func TestShellSplitterInvalid(t *testing.T) {
 				t.Fatalf("expected error not found, want:\n  %q\ngot:\n  %q", test.err, err.Error())
 			}
 
-			if cmd != "" {
-				t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
-			}
-
 			if len(args) > 0 {
 				t.Fatalf("splitter returned fields from invalid data: %v", args)
 			}

internal/backend/test/tests.go

@@ -147,6 +147,10 @@ func (s *Suite) TestLoad(t *testing.T) {
 	}
 
 	err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error {
+		_, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
 		return errors.Errorf("deliberate error")
 	})
 	if err == nil {

run_integration_tests.go

@@ -97,6 +97,7 @@ func (env *TravisEnvironment) Prepare() error {
 		"github.com/golang/dep/cmd/dep",
 		"github.com/restic/rest-server/cmd/rest-server",
 		"github.com/restic/calens",
+		"github.com/ncw/rclone",
 	}
 
 	for _, pkg := range pkgs {
@@ -191,6 +192,7 @@ func (env *TravisEnvironment) RunTests() error {
 		"restic/backend/rest.TestBackendREST",
 		"restic/backend/sftp.TestBackendSFTP",
 		"restic/backend/s3.TestBackendMinio",
+		"restic/backend/rclone.TestBackendRclone",
 	}
 
 	// if the test s3 repository is available, make sure that the test is not skipped