alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

backend/rest: Implement REST API v2

Browse source
This commit is contained in:
Alexander Neumann 2018-01-23 23:12:52 +01:00
parent 0f4cbea27d
commit 7e6bfdae79
2 changed files with 231 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
internal/backend/rest/rest.go
View file

@ -30,6 +30,11 @@ type restBackend struct {
backend.Layout
}
const (
contentTypeV1 = "application/vnd.x.restic.rest.v1"
contentTypeV2 = "application/vnd.x.restic.rest.v2"
)
// Open opens the REST backend with the given config.
func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
client := &http.Client{Transport: rt}
@ -111,8 +116,15 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (
// make sure that client.Post() cannot close the reader by wrapping it
rd = ioutil.NopCloser(rd)
req, err := http.NewRequest(http.MethodPost, b.Filename(h), rd)
if err != nil {
return errors.Wrap(err, "NewRequest")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", contentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd)
resp, err := ctxhttp.Do(ctx, b.client, req)
b.sem.ReleaseToken()
if resp != nil {
@ -180,7 +192,8 @@ func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, off
if length > 0 {
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
}
req.Header.Add("Range", byteRange)
req.Header.Set("Range", byteRange)
req.Header.Set("Accept", contentTypeV2)
debug.Log("Load(%v) send range %v", h, byteRange)
b.sem.GetToken()
@ -214,8 +227,14 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
return restic.FileInfo{}, err
}
req, err := http.NewRequest(http.MethodHead, b.Filename(h), nil)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h))
resp, err := ctxhttp.Do(ctx, b.client, req)
b.sem.ReleaseToken()
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
@ -267,6 +286,8 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
if err != nil {
return errors.Wrap(err, "http.NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Do(ctx, b.client, req)
b.sem.ReleaseToken()
@ -300,17 +321,35 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
url += "/"
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return errors.Wrap(err, "NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Get(ctx, b.client, url)
resp, err := ctxhttp.Do(ctx, b.client, req)
b.sem.ReleaseToken()
if err != nil {
return errors.Wrap(err, "Get")
}
if resp.Header.Get("Content-Type") == contentTypeV2 {
return b.listv2(ctx, t, resp, fn)
}
return b.listv1(ctx, t, resp, fn)
}
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file.
func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body)
var list []string
if err = dec.Decode(&list); err != nil {
if err := dec.Decode(&list); err != nil {
return errors.Wrap(err, "Decode")
}
@ -338,6 +377,43 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
return ctx.Err()
}
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files.
func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body)
var list []struct {
Name string `json:"name"`
Size int64 `json:"size"`
}
if err := dec.Decode(&list); err != nil {
return errors.Wrap(err, "Decode")
}
for _, item := range list {
if ctx.Err() != nil {
return ctx.Err()
}
fi := restic.FileInfo{
Name: item.Name,
Size: item.Size,
}
err := fn(fi)
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
}
return ctx.Err()
}
// Close closes all open files.
func (b *restBackend) Close() error {
// this does not need to do anything, all open files are closed within the

150
internal/backend/rest/rest_int_test.go Normal file
View file

@ -0,0 +1,150 @@
package rest_test
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strconv"
"testing"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/restic"
)
func TestListAPI(t *testing.T) {
var tests = []struct {
Name string
ContentType string // response header
Data string // response data
Requests int
Result []restic.FileInfo
}{
{
Name: "content-type-unknown",
ContentType: "application/octet-stream",
Data: `[
"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
},
Requests: 4,
},
{
Name: "content-type-v1",
ContentType: "application/vnd.x.restic.rest.v1",
Data: `[
"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
},
Requests: 4,
},
{
Name: "content-type-v2",
ContentType: "application/vnd.x.restic.rest.v2",
Data: `[
{"name": "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", "size": 1001},
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
]`,
Result: []restic.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
},
Requests: 1,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
numRequests := 0
srv := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
numRequests++
t.Logf("req %v %v, accept: %v", req.Method, req.URL.Path, req.Header["Accept"])
var err error
switch {
case req.Method == "GET":
// list files in data/
res.Header().Set("Content-Type", test.ContentType)
_, err = res.Write([]byte(test.Data))
if err != nil {
t.Fatal(err)
}
return
case req.Method == "HEAD":
// stat file in data/, use the first two bytes in the name
// of the file as the size :)
filename := req.URL.Path[6:]
len, err := strconv.ParseInt(filename[:4], 16, 64)
if err != nil {
t.Fatal(err)
}
res.Header().Set("Content-Length", fmt.Sprintf("%d", len))
res.WriteHeader(http.StatusOK)
return
}
t.Errorf("unhandled request %v %v", req.Method, req.URL.Path)
}))
defer srv.Close()
srvURL, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
cfg := rest.Config{
Connections: 5,
URL: srvURL,
}
be, err := rest.Open(cfg, http.DefaultTransport)
if err != nil {
t.Fatal(err)
}
var list []restic.FileInfo
err = be.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error {
list = append(list, fi)
return nil
})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(list, test.Result) {
t.Fatalf("wrong response returned, want:\n %v\ngot: %v", test.Result, list)
}
if numRequests != test.Requests {
t.Fatalf("wrong number of HTTP requests executed, want %d, got %d", test.Requests, numRequests)
}
defer func() {
err = be.Close()
if err != nil {
t.Fatal(err)
}
}()
})
}
}