Implement Remote Control for rclone #2111

This implements a remote control protocol activated with the --rc flag
and a new command `rclone rc` to use that interface.

Still to do
  * docs - need finishing
  * tests

bin/make_manual.py

@@ -18,6 +18,7 @@ docs = [
     "docs.md",
     "remote_setup.md",
     "filtering.md",
+    "rc.md",
     "overview.md",
 
     # Keep these alphabetical by full name

cmd/all/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/ncdu"
 	_ "github.com/ncw/rclone/cmd/obscure"
 	_ "github.com/ncw/rclone/cmd/purge"
+	_ "github.com/ncw/rclone/cmd/rc"
 	_ "github.com/ncw/rclone/cmd/rcat"
 	_ "github.com/ncw/rclone/cmd/rmdir"
 	_ "github.com/ncw/rclone/cmd/rmdirs"

cmd/cmd.go

@@ -30,6 +30,8 @@ import (
 	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/fspath"
 	fslog "github.com/ncw/rclone/fs/log"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/fs/rc/rcflags"
 	"github.com/ncw/rclone/lib/atexit"
 )
 
@@ -126,6 +128,7 @@ func init() {
 	// Add global flags
 	configflags.AddFlags(pflag.CommandLine)
 	filterflags.AddFlags(pflag.CommandLine)
+	rcflags.AddFlags(pflag.CommandLine)
 
 	Root.Run = runRoot
 	Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
@@ -383,6 +386,9 @@ func initConfig() {
 	// Write the args for debug purposes
 	fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
 
+	// Start the remote control if configured
+	rc.Start(&rcflags.Opt)
+
 	// Setup CPU profiling if desired
 	if *cpuProfile != "" {
 		fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)

cmd/rc/rc.go

@@ -0,0 +1,139 @@
+package rc
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	noOutput = false
+	url      = "http://localhost:5572/"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.")
+	commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.")
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "rc commands parameter",
+	Short: `Run a command against a running rclone.`,
+	Long: `
+This runs a command against a running rclone.  By default it will use
+that specified in the --rc-addr command.
+
+Arguments should be passed in as parameter=value.
+
+The result will be returned as a JSON object by default.
+
+Use "rclone rc list" to see a list of all possible commands.`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 1E9, command, args)
+		cmd.Run(false, false, command, func() error {
+			if len(args) == 0 {
+				return list()
+			}
+			return run(args)
+		})
+	},
+}
+
+// do a call from (path, in) to (out, err).
+//
+// if err is set, out may be a valid error return or it may be nil
+func doCall(path string, in rc.Params) (out rc.Params, err error) {
+	// Do HTTP request
+	client := fshttp.NewClient(fs.Config)
+	url := url
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+	url += path
+	data, err := json.Marshal(in)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to encode JSON")
+	}
+	resp, err := client.Post(url, "application/json", bytes.NewBuffer(data))
+	if err != nil {
+		return nil, errors.Wrap(err, "connection failed")
+	}
+	defer fs.CheckClose(resp.Body, &err)
+
+	// Parse output
+	out = make(rc.Params)
+	err = json.NewDecoder(resp.Body).Decode(&out)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode JSON")
+	}
+
+	// Check we got 200 OK
+	if resp.StatusCode != http.StatusOK {
+		err = errors.Errorf("operation %q failed: %v", path, out["error"])
+	}
+
+	return out, err
+}
+
+// Run the remote control command passed in
+func run(args []string) (err error) {
+	path := strings.Trim(args[0], "/")
+
+	// parse input
+	in := make(rc.Params)
+	for _, param := range args[1:] {
+		equals := strings.IndexRune(param, '=')
+		if equals < 0 {
+			return errors.Errorf("No '=' found in parameter %q", param)
+		}
+		key, value := param[:equals], param[equals+1:]
+		in[key] = value
+	}
+
+	// Do the call
+	out, callErr := doCall(path, in)
+
+	// Write the JSON blob to stdout if required
+	if out != nil && !noOutput {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "\t")
+		err = enc.Encode(out)
+		if err != nil {
+			return errors.Wrap(err, "failed to output JSON")
+		}
+	}
+
+	return callErr
+}
+
+// List the available commands to stdout
+func list() error {
+	list, err := doCall("rc/list", nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to list")
+	}
+	commands, ok := list["commands"].([]interface{})
+	if !ok {
+		return errors.New("bad JSON")
+	}
+	for _, command := range commands {
+		info, ok := command.(map[string]interface{})
+		if !ok {
+			return errors.New("bad JSON")
+		}
+		fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"])
+		fmt.Printf("%s\n\n", info["Help"])
+	}
+	return nil
+}

docs/content/docs.md

@@ -985,6 +985,16 @@ For the filtering options
 
 See the [filtering section](/filtering/).
 
+Remote control
+--------------
+
+For the remote control options and for instructions on how to remote control rclone
+
+  * `--rc`
+  * and anything starting with `--rc-`
+
+See [the remote control section](/rc/).
+
 Logging
 -------
 

docs/content/rc.md

@@ -0,0 +1,145 @@
+---
+title: "Remote Control"
+description: "Remote controlling rclone"
+date: "2018-03-05"
+---
+
+# Remote controlling rclone #
+
+If rclone is run with the `--rc` flag then it starts an http server
+which can be used to remote control rclone.
+
+FIXME describe other flags
+
+## Accessing the remote control via the rclone rc command
+
+Rclone itself implements the remote control protocol in its `rclone
+rc` command.
+
+You can use it like this
+
+
+
+## Accessing the remote control via HTTP
+
+Rclone implements a simple HTTP based protocol.
+
+Each endpoint takes an JSON object and returns a JSON object or an
+error.  The JSON objects are essentially a map of string names to
+values.
+
+All calls must made using POST.
+
+The input objects can be supplied using URL parameters, POST
+parameters or by supplying "Content-Type: application/json" and a JSON
+blob in the body.  There are examples of these below using `curl`.
+
+The response will be a JSON blob in the body of the response.  This is
+formatted to be reasonably human readable.
+
+If an error occurs then there will be an HTTP error status (usually
+400) and the body of the response will contain a JSON encoded error
+object.
+
+### Using POST with URL parameters only
+
+```
+curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2'
+```
+
+Response
+
+```
+{
+	"potato": "1",
+	"sausage": "2"
+}
+```
+
+Here is what an error response looks like:
+
+```
+curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+```
+
+```
+{
+	"error": "arbitrary error on input map[potato:1 sausage:2]",
+	"input": {
+		"potato": "1",
+		"sausage": "2"
+	}
+}
+```
+
+Note that curl doesn't return errors to the shell unless you use the `-f` option
+
+```
+$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+curl: (22) The requested URL returned error: 400 Bad Request
+$ echo $?
+22
+```
+
+### Using POST with a form
+
+```
+curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/
+```
+
+Response
+
+```
+{
+	"potato": "1",
+	"sausage": "2"
+}
+```
+
+Note that you can combine these with URL parameters too with the POST
+parameters taking precedence.
+
+```
+curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4"
+```
+
+Response
+
+```
+{
+	"potato": "1",
+	"rutabaga": "3",
+	"sausage": "4"
+}
+
+```
+
+### Using POST with a JSON blob
+
+```
+curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/
+```
+
+response
+
+```
+{
+	"password": "xyz",
+	"username": "xyz"
+}
+```
+
+This can be combined with URL parameters too if required.  The JSON
+blob takes precedence.
+
+```
+curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4'
+```
+
+```
+{
+	"potato": 2,
+	"rutabaga": "3",
+	"sausage": 1
+}
+```

docs/layouts/chrome/navbar.html

@@ -18,6 +18,7 @@
                     <li><a href="/install/"><i class="fa fa-book"></i> Installation</a></li>
                     <li><a href="/docs/"><i class="fa fa-book"></i> Usage</a></li>
                     <li><a href="/filtering/"><i class="fa fa-book"></i> Filtering</a></li>
+                    <li><a href="/rc/"><i class="fa fa-book"></i> Remote Control</a></li>
                     <li><a href="/changelog/"><i class="fa fa-book"></i> Changelog</a></li>
                     <li><a href="/bugs/"><i class="fa fa-book"></i> Bugs</a></li>
                     <li><a href="/faq/"><i class="fa fa-book"></i> FAQ</a></li>

fs/rc/internal.go

@@ -0,0 +1,50 @@
+// Define the internal rc functions
+
+package rc
+
+import "github.com/pkg/errors"
+
+func init() {
+	Add(Call{
+		Path:  "rc/noop",
+		Fn:    rcNoop,
+		Title: "Echo the input to the output parameters",
+		Help: `
+This echoes the input parameters to the output parameters for testing
+purposes.  It can be used to check that rclone is still alive and to
+check that parameter passing is working properly.`,
+	})
+	Add(Call{
+		Path:  "rc/error",
+		Fn:    rcError,
+		Title: "This returns an error",
+		Help: `
+This returns an error with the input as part of its error string.
+Useful for testing error handling.`,
+	})
+	Add(Call{
+		Path:  "rc/list",
+		Fn:    rcList,
+		Title: "List all the registered remote control commands",
+		Help: `
+This lists all the registered remote control commands as a JSON map in
+the commands response.`,
+	})
+}
+
+// Echo the input to the ouput parameters
+func rcNoop(in Params) (out Params, err error) {
+	return in, nil
+}
+
+// Return an error regardless
+func rcError(in Params) (out Params, err error) {
+	return nil, errors.Errorf("arbitrary error on input %+v", in)
+}
+
+// List the registered commands
+func rcList(in Params) (out Params, err error) {
+	out = make(Params)
+	out["commands"] = registry.list()
+	return out, nil
+}

fs/rc/rc.go

@@ -0,0 +1,135 @@
+// Package rc implements a remote control server and registry for rclone
+//
+// To register your internal calls, call rc.Add(path, function).  Your
+// function should take ane return a Param.  It can also return an
+// error.  Use rc.NewError to wrap an existing error along with an
+// http response type if another response other than 500 internal
+// error is required on error.
+package rc
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/ncw/rclone/cmd/serve/httplib"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// Options contains options for the remote control server
+type Options struct {
+	HTTPOptions httplib.Options
+	Enabled     bool
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	HTTPOptions: httplib.DefaultOpt,
+	Enabled:     false,
+}
+
+func init() {
+	DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
+}
+
+// Start the remote control server if configured
+func Start(opt *Options) {
+	if opt.Enabled {
+		s := newServer(opt)
+		go s.serve()
+	}
+}
+
+// server contains everything to run the server
+type server struct {
+	srv *httplib.Server
+}
+
+func newServer(opt *Options) *server {
+	// Serve on the DefaultServeMux so can have global registrations appear
+	mux := http.DefaultServeMux
+	s := &server{
+		srv: httplib.NewServer(mux, &opt.HTTPOptions),
+	}
+	mux.HandleFunc("/", s.handler)
+	return s
+}
+
+// serve runs the http server - doesn't return
+func (s *server) serve() {
+	fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
+	s.srv.Serve()
+}
+
+// writes JSON in out to w
+func writeJSON(w http.ResponseWriter, out Params) {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "\t")
+	err := enc.Encode(out)
+	if err != nil {
+		// can't return the error at this point
+		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+	}
+}
+
+// handler reads incoming requests and dispatches them
+func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+	path := strings.Trim(r.URL.Path, "/")
+	in := make(Params)
+
+	writeError := func(err error, status int) {
+		fs.Errorf(nil, "rc: %q: error: %v", path, err)
+		w.WriteHeader(status)
+		writeJSON(w, Params{
+			"error": err.Error(),
+			"input": in,
+		})
+	}
+
+	if r.Method != "POST" {
+		writeError(errors.Errorf("method %q not allowed - POST required", r.Method), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Find the call
+	call := registry.get(path)
+	if call == nil {
+		writeError(errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Parse the POST and URL parameters into r.Form
+	err := r.ParseForm()
+	if err != nil {
+		writeError(errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
+		return
+	}
+
+	// Read the POST and URL parameters into in
+	for k, vs := range r.Form {
+		if len(vs) > 0 {
+			in[k] = vs[len(vs)-1]
+		}
+	}
+	fs.Debugf(nil, "form = %+v", r.Form)
+
+	// Parse a JSON blob from the input
+	if r.Header.Get("Content-Type") == "application/json" {
+		err := json.NewDecoder(r.Body).Decode(&in)
+		if err != nil {
+			writeError(errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
+			return
+		}
+	}
+
+	fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
+	out, err := call.Fn(in)
+	if err != nil {
+		writeError(errors.Wrap(err, "remote control command failed"), http.StatusInternalServerError)
+		return
+	}
+
+	fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
+	writeJSON(w, out)
+}

fs/rc/rcflags/rcflags.go

@@ -0,0 +1,20 @@
+// Package rcflags implements command line flags to set up the remote control
+package rcflags
+
+import (
+	"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
+	"github.com/ncw/rclone/fs/config/flags"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/spf13/pflag"
+)
+
+// Options set by command line flags
+var (
+	Opt = rc.DefaultOpt
+)
+
+// AddFlags adds the remote control flags to the flagSet
+func AddFlags(flagSet *pflag.FlagSet) {
+	flags.BoolVarP(flagSet, &Opt.Enabled, "rc", "", false, "Enable the remote control server.")
+	httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions)
+}

fs/rc/registry.go

@@ -0,0 +1,73 @@
+// Define the registry
+
+package rc
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+)
+
+// Params is the input and output type for the Func
+type Params map[string]interface{}
+
+// Func defines a type for a remote control function
+type Func func(in Params) (out Params, err error)
+
+// Call defines info about a remote control function and is used in
+// the Add function to create new entry points.
+type Call struct {
+	Path  string // path to activate this RC
+	Fn    Func   `json:"-"` // function to call
+	Title string // help for the function
+	Help  string // multi-line markdown formatted help
+}
+
+// Registry holds the list of all the registered remote control functions
+type Registry struct {
+	mu   sync.RWMutex
+	call map[string]*Call
+}
+
+// NewRegistry makes a new registry for remote control functions
+func NewRegistry() *Registry {
+	return &Registry{
+		call: make(map[string]*Call),
+	}
+}
+
+// Add a call to the registry
+func (r *Registry) add(call Call) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	call.Path = strings.Trim(call.Path, "/")
+	call.Help = strings.TrimSpace(call.Help)
+	fs.Debugf(nil, "Adding path %q to remote control registry", call.Path)
+	r.call[call.Path] = &call
+}
+
+// get a Call from a path or nil
+func (r *Registry) get(path string) *Call {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.call[path]
+}
+
+// get a list of all calls
+func (r *Registry) list() (out []*Call) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	for _, call := range r.call {
+		out = append(out, call)
+	}
+	return out
+}
+
+// The global registry
+var registry = NewRegistry()
+
+// Add a function to the global registry
+func Add(call Call) {
+	registry.add(call)
+}