From d073efdc6ce90e55839b8feb7cc1852acfb80b1a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 17 Feb 2018 23:25:59 +0000 Subject: [PATCH] serve restic: serves a remote in restic REST API format --- cmd/serve/restic/restic.go | 392 ++++++++++++++++++++++++++++++++ cmd/serve/restic/restic_test.go | 95 ++++++++ cmd/serve/serve.go | 2 + 3 files changed, 489 insertions(+) create mode 100644 cmd/serve/restic/restic.go create mode 100644 cmd/serve/restic/restic_test.go diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go new file mode 100644 index 000000000..c00e2db03 --- /dev/null +++ b/cmd/serve/restic/restic.go @@ -0,0 +1,392 @@ +// Package restic serves a remote suitable for use with restic +package restic + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/cmd/serve/httplib" + "github.com/ncw/rclone/cmd/serve/httplib/httpflags" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/object" + "github.com/ncw/rclone/fs/operations" + "github.com/ncw/rclone/fs/walk" + "github.com/spf13/cobra" +) + +func init() { + httpflags.AddFlags(Command.Flags()) +} + +// Command definition for cobra +var Command = &cobra.Command{ + Use: "restic remote:path", + Short: `Serve the remote for restic's REST API.`, + Long: `rclone serve restic implements restic's REST backend API +over HTTP. This allows restic to use rclone as a data storage +mechanism for cloud providers that restic does not support directly. + +[Restic](https://restic.net/) is a command line program for doing +backups. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to +control the stats printing. + +### Setting up rclone for use by restic ### + +First [set up a remote for your chosen cloud provider](/docs/#configure). + +Once you have set up the remote, check it is working with, for example +"rclone lsd remote:". You may have called the remote something other +than "remote:" - just substitute whatever you called it in the +following instructions. + +Now start the rclone restic server + + rclone serve restic -v remote:backup + +Where you can replace "backup" in the above by whatever path in the +remote you wish to use. + +By default this will serve on "localhost:8080" you can change this +with use of the "--addr" flag. + +You might wish to start this server on boot. + +### Setting up restic to use rclone ### + +Now you can [follow the restic +instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) +on setting up restic. + +Note that you will need restic 0.8.2 or later to interoperate with +rclone. + +For the example above you will want to use "http://localhost:8080/" as +the URL for the REST server. + +For example: + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/ + $ export RESTIC_PASSWORD=yourpassword + $ restic init + created restic backend 8b1a4b56ae at rest:http://localhost:8080/ + + Please note that knowledge of your password is required to access + the repository. Losing your password means that your data is + irrecoverably lost. + $ restic backup /path/to/files/to/backup + scan [/path/to/files/to/backup] + scanned 189 directories, 312 files in 0:00 + [0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00 + duration: 0:00 + snapshot 45c8fdd8 saved + +#### Multiple repositories #### + +Note that you can use the endpoint to host multiple repositories. Do +this by adding a directory name or path after the URL. Note that +these **must** end with /. Eg + + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/ + # backup user1 stuff + $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ + # backup user2 stuff + +` + httplib.Help, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1, command, args) + f := cmd.NewFsSrc(args) + cmd.Run(false, true, command, func() error { + s := newServer(f, &httpflags.Opt) + s.serve() + return nil + }) + }, +} + +// server contains everything to run the server +type server struct { + f fs.Fs + srv *httplib.Server +} + +func newServer(f fs.Fs, opt *httplib.Options) *server { + mux := http.NewServeMux() + s := &server{ + f: f, + srv: httplib.NewServer(mux, opt), + } + mux.HandleFunc("/", s.handler) + return s +} + +// serve runs the http server - doesn't return +func (s *server) serve() { + fs.Logf(s.f, "Serving restic REST API on %s", s.srv.URL()) + s.srv.Serve() +} + +var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$") + +// Makes a remote from a URL path. This implements the backend layout +// required by restic. +func makeRemote(path string) string { + path = strings.Trim(path, "/") + parts := matchData.FindStringSubmatch(path) + // if no data directory, layout is flat + if parts == nil { + return path + } + // otherwise map + // data/2159dd48 to + // data/21/2159dd48 + fileName := parts[1] + prefix := path[:len(path)-len(fileName)] + return prefix + fileName[:2] + "/" + fileName +} + +// handler reads incoming requests and dispatches them +func (s *server) handler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Server", "rclone/"+fs.Version) + + path := r.URL.Path + remote := makeRemote(path) + fs.Debugf(s.f, "%s %s", r.Method, path) + + // Dispatch on path then method + if strings.HasSuffix(path, "/") { + switch r.Method { + case "GET": + s.listObjects(w, r, remote) + case "POST": + s.createRepo(w, r, remote) + default: + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + } + } else { + switch r.Method { + case "GET": + s.getObject(w, r, remote) + case "HEAD": + s.headObject(w, r, remote) + case "POST": + s.postObject(w, r, remote) + case "DELETE": + s.deleteObject(w, r, remote) + default: + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + } + } +} + +// head request the remote +func (s *server) headObject(w http.ResponseWriter, r *http.Request, remote string) { + o, err := s.f.NewObject(remote) + if err != nil { + fs.Debugf(remote, "Head request error: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Set content length since we know how long the object is + w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10)) +} + +// get the remote +func (s *server) getObject(w http.ResponseWriter, r *http.Request, remote string) { + o, err := s.f.NewObject(remote) + if err != nil { + fs.Debugf(remote, "Get request error: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Set content length since we know how long the object is + w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10)) + + // Decode Range request if present + code := http.StatusOK + size := o.Size() + var options []fs.OpenOption + if rangeRequest := r.Header.Get("Range"); rangeRequest != "" { + //fs.Debugf(nil, "Range: request %q", rangeRequest) + option, err := fs.ParseRangeOption(rangeRequest) + if err != nil { + fs.Debugf(remote, "Get request parse range request error: %v", err) + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + options = append(options, option) + offset, limit := option.Decode(o.Size()) + end := o.Size() // exclusive + if limit >= 0 { + end = offset + limit + } + if end > o.Size() { + end = o.Size() + } + size = end - offset + // fs.Debugf(nil, "Range: offset=%d, limit=%d, end=%d, size=%d (object size %d)", offset, limit, end, size, o.Size()) + // Content-Range: bytes 0-1023/146515 + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end-1, o.Size())) + // fs.Debugf(nil, "Range: Content-Range: %q", w.Header().Get("Content-Range")) + code = http.StatusPartialContent + } + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + + file, err := o.Open(options...) + if err != nil { + fs.Debugf(remote, "Get request open error: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + defer func() { + err := file.Close() + if err != nil { + fs.Errorf(remote, "Get request: close failed: %v", err) + } + }() + + w.WriteHeader(code) + + _, err = io.Copy(w, file) + if err != nil { + fs.Errorf(remote, "Didn't finish writing GET request: %v", err) + return + } +} + +// postObject posts an object to the repository +func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote string) { + // fs.Debugf(s.f, "content length = %d", r.ContentLength) + if r.ContentLength >= 0 { + // Size known use Put + info := object.NewStaticObjectInfo(remote, time.Now(), r.ContentLength, true, nil, s.f) + _, err := s.f.Put(r.Body, info) + if err != nil { + fs.Errorf(remote, "Post request put error: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + } else { + // Size unknown use Rcat + _, err := operations.Rcat(s.f, remote, r.Body, time.Now()) + if err != nil { + fs.Errorf(remote, "Post request rcat error: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + } +} + +// delete the remote +func (s *server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) { + o, err := s.f.NewObject(remote) + if err != nil { + fs.Debugf(remote, "Delete request error: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + if err := o.Remove(); err != nil { + fs.Errorf(remote, "Delete request remove error: %v", err) + if err == fs.ErrorObjectNotFound { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + } else { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + return + } +} + +// listItem is an element returned for the restic v2 list response +type listItem struct { + Name string `json:"name"` + Size int64 `json:"size"` +} + +// return type for list +type listItems []listItem + +// add a DirEntry to the listItems +func (ls *listItems) add(entry fs.DirEntry) { + if o, ok := entry.(fs.Object); ok { + *ls = append(*ls, listItem{ + Name: path.Base(o.Remote()), + Size: o.Size(), + }) + } +} + +// listObjects lists all Objects of a given type in an arbitrary order. +func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote string) { + fs.Debugf(remote, "list request") + + var ls listItems + err := walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error { + if err == nil { + for _, entry := range entries { + ls.add(entry) + } + } + return err + }) + if err != nil { + fs.Errorf(remote, "list failed: %v", err) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + data, err := json.Marshal(ls) + if err != nil { + fs.Errorf(remote, "list marshal failed: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + // fs.Debugf(remote, "ls result = %s", data) + + w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2") + + _, _ = w.Write(data) +} + +// createRepo creates repository directories. +// +// We don't bother creating the data dirs as rclone will create them on the fly +func (s *server) createRepo(w http.ResponseWriter, r *http.Request, remote string) { + fs.Infof(remote, "Creating repository") + + if r.URL.Query().Get("create") != "true" { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + err := s.f.Mkdir(remote) + if err != nil { + fs.Errorf(remote, "Create repo failed to Mkdir: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} { + dirRemote := path.Join(remote, name) + err := s.f.Mkdir(dirRemote) + if err != nil { + fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + } +} diff --git a/cmd/serve/restic/restic_test.go b/cmd/serve/restic/restic_test.go new file mode 100644 index 000000000..cb8ae28e2 --- /dev/null +++ b/cmd/serve/restic/restic_test.go @@ -0,0 +1,95 @@ +// Serve restic tests set up a server and run the integration tests +// for restic against it. +package restic + +import ( + "os" + "os/exec" + "testing" + + _ "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/cmd/serve/httplib" + "github.com/ncw/rclone/fstest" + "github.com/stretchr/testify/assert" +) + +const ( + testBindAddress = "localhost:51779" + testURL = "http://" + testBindAddress + "/" + resticSource = "../../../../../restic/restic" +) + +// TestRestic runs the restic server then runs the unit tests for the +// restic remote against it. +func TestRestic(t *testing.T) { + _, err := os.Stat(resticSource) + if err != nil { + t.Skipf("Skipping test as restic source not found: %v", err) + } + + opt := httplib.DefaultOpt + opt.ListenAddr = testBindAddress + + fstest.Initialise() + + fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) + assert.NoError(t, err) + defer clean() + + err = fremote.Mkdir("") + assert.NoError(t, err) + + // Start the server + w := newServer(fremote, &opt) + go w.serve() + defer w.srv.Close() + + // Change directory to run the tests + err = os.Chdir(resticSource) + assert.NoError(t, err, "failed to cd to restic source code") + + // Run the restic tests + runTests := func(path string) { + args := []string{"test", "./internal/backend/rest", "-run", "TestBackendRESTExternalServer", "-count=1"} + if testing.Verbose() { + args = append(args, "-v") + } + cmd := exec.Command("go", args...) + cmd.Env = append(os.Environ(), + "RESTIC_TEST_REST_REPOSITORY=rest:"+testURL+path, + ) + out, err := cmd.CombinedOutput() + if len(out) != 0 { + t.Logf("\n----------\n%s----------\n", string(out)) + } + assert.NoError(t, err, "Running restic integration tests") + } + + // Run the tests with no path + runTests("") + //... and again with a path + runTests("potato/sausage/") + +} + +func TestMakeRemote(t *testing.T) { + for _, test := range []struct { + in, want string + }{ + {"", ""}, + {"/", ""}, + {"/data", "data"}, + {"/data/", "data"}, + {"/data/1", "data/1"}, + {"/data/12", "data/12/12"}, + {"/data/123", "data/12/123"}, + {"/data/123/", "data/12/123"}, + {"/keys", "keys"}, + {"/keys/1", "keys/1"}, + {"/keys/12", "keys/12"}, + {"/keys/123", "keys/123"}, + } { + got := makeRemote(test.in) + assert.Equal(t, test.want, got, test.in) + } +} diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 5314c4a8a..795828db5 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -5,6 +5,7 @@ import ( "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/serve/http" + "github.com/ncw/rclone/cmd/serve/restic" "github.com/ncw/rclone/cmd/serve/webdav" "github.com/spf13/cobra" ) @@ -12,6 +13,7 @@ import ( func init() { Command.AddCommand(http.Command) Command.AddCommand(webdav.Command) + Command.AddCommand(restic.Command) cmd.Root.AddCommand(Command) }