KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #1914 from restic/update-blazer

Browse source 
Add support for B2 application keys
This commit is contained in:
Alexander Neumann 2018-07-31 21:27:50 +02:00
commit 7260110c27
11 changed files with 285 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Gopkg.lock generated
View file

@ -94,8 +94,8 @@
[[projects]]
name = "github.com/kurin/blazer"
packages = ["b2","base","internal/b2assets","internal/b2types","internal/blog","x/window"]
revision = "7f1134c7489e86be5c924137996d4e421815f48a"
version = "v0.5.0"
revision = "caf65aa76491dc533bac68ad3243ce72fa4e0a0a"
version = "v0.5.1"
[[projects]]
name = "github.com/marstr/guid"

13
doc/030_preparing_a_new_repo.rst
View file

@ -293,16 +293,21 @@ Backblaze B2
************
Restic can backup data to any Backblaze B2 bucket. You need to first setup the
following environment variables with the credentials you obtained when signed
into your B2 account:
following environment variables with the credentials you can find in the
dashboard in on the "Buckets" page when signed into your B2 account:
.. code-block:: console
$ export B2_ACCOUNT_ID=<MY_ACCOUNT_ID>
$ export B2_ACCOUNT_KEY=<MY_SECRET_ACCOUNT_KEY>
You can then easily initialize a repository stored at Backblaze B2. If the
bucket does not exist yet, it will be created:
You can either specify the so-called "Master Application Key" here (which can
access any bucket at any path) or a dedicated "Application Key" created just
for restic (which may be restricted to a specific bucket and/or path).
You can then initialize a repository stored at Backblaze B2. If the
bucket does not exist yet and the credentials you passed to restic have the
privilege to create buckets, it will be created automatically:
.. code-block:: console

2
vendor/github.com/kurin/blazer/b2/backend.go generated vendored
View file

@ -154,6 +154,7 @@ type beKeyInterface interface {
name() string
expires() time.Time
secret() string
id() string
}
type beKey struct {
@ -711,6 +712,7 @@ func (b *beKey) caps() []string { return b.k.caps() }
func (b *beKey) name() string { return b.k.name() }
func (b *beKey) expires() time.Time { return b.k.expires() }
func (b *beKey) secret() string { return b.k.secret() }
func (b *beKey) id() string { return b.k.id() }
func jitter(d time.Duration) time.Duration {
f := float64(d)

2
vendor/github.com/kurin/blazer/b2/baseline.go generated vendored
View file

@ -105,6 +105,7 @@ type b2KeyInterface interface {
name() string
expires() time.Time
secret() string
id() string
}
type b2Root struct {
@ -508,3 +509,4 @@ func (b *b2Key) caps() []string { return b.b.Capabilities }
func (b *b2Key) name() string { return b.b.Name }
func (b *b2Key) expires() time.Time { return b.b.Expires }
func (b *b2Key) secret() string { return b.b.Secret }
func (b *b2Key) id() string { return b.b.ID }

66
vendor/github.com/kurin/blazer/b2/integration_test.go generated vendored
View file

@ -1014,6 +1014,68 @@ func TestVerifyReader(t *testing.T) {
}
}
func TestListBucketsWithKey(t *testing.T) {
ctx := context.Background()
bucket, done := startLiveTest(ctx, t)
defer done()
key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets"))
if err != nil {
t.Fatal(err)
}
client, err := NewClient(ctx, key.ID(), key.Secret())
if err != nil {
t.Fatal(err)
}
if _, err := client.Bucket(ctx, bucket.Name()); err != nil {
t.Fatal(err)
}
}
func TestListBucketContentsWithKey(t *testing.T) {
ctx := context.Background()
bucket, done := startLiveTest(ctx, t)
defer done()
for _, path := range []string{"foo/bar", "foo/baz", "foo", "bar", "baz"} {
if _, _, err := writeFile(ctx, bucket, path, 1, 1e8); err != nil {
t.Fatal(err)
}
}
key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets", "listFiles"), Prefix("foo/"))
if err != nil {
t.Fatal(err)
}
client, err := NewClient(ctx, key.ID(), key.Secret())
if err != nil {
t.Fatal(err)
}
obucket, err := client.Bucket(ctx, bucket.Name())
if err != nil {
t.Fatal(err)
}
iter := obucket.List(ctx)
var got []string
for iter.Next() {
got = append(got, iter.Object().Name())
}
if iter.Err() != nil {
t.Fatal(iter.Err())
}
want := []string{"foo/bar", "foo/baz"}
if !reflect.DeepEqual(got, want) {
t.Errorf("error listing objects with restricted key: got %v, want %v", got, want)
}
iter2 := obucket.List(ctx, ListHidden())
for iter2.Next() {
}
if iter2.Err() != nil {
t.Error(iter2.Err())
}
}
func TestCreateDeleteKey(t *testing.T) {
ctx := context.Background()
bucket, done := startLiveTest(ctx, t)
@ -1049,9 +1111,7 @@ func TestCreateDeleteKey(t *testing.T) {
for _, e := range table {
var opts []KeyOption
for _, cap := range e.cap {
opts = append(opts, Capability(cap))
}
opts = append(opts, Capabilities(e.cap...))
if e.d != 0 {
opts = append(opts, Lifetime(e.d))
}

10
vendor/github.com/kurin/blazer/b2/key.go generated vendored
View file

@ -47,6 +47,10 @@ func (k *Key) Delete(ctx context.Context) error { return k.k.del(ctx) }
// operations.
func (k *Key) Secret() string { return k.k.secret() }
// ID returns the application key ID. This, plus the secret, is necessary to
// authenticate to B2.
func (k *Key) ID() string { return k.k.id() }
type keyOptions struct {
caps []string
prefix string
@ -69,10 +73,10 @@ func Deadline(t time.Time) KeyOption {
return Lifetime(d)
}
// Capability requests a key with the given capability.
func Capability(cap string) KeyOption {
// Capabilities requests a key with the given capability.
func Capabilities(caps ...string) KeyOption {
return func(k *keyOptions) {
k.caps = append(k.caps, cap)
k.caps = append(k.caps, caps...)
}
}

13
vendor/github.com/kurin/blazer/base/base.go generated vendored
View file

@ -42,7 +42,7 @@ import (
const (
APIBase = "https://api.backblazeb2.com"
DefaultUserAgent = "blazer/0.5.0"
DefaultUserAgent = "blazer/0.5.1"
)
type b2err struct {
@ -268,6 +268,8 @@ type B2 struct {
downloadURI string
minPartSize int
opts *b2Options
bucket string // restricted to this bucket if present
pfx string // restricted to objects with this prefix if present
}
// Update replaces the B2 object with a new one, in-place.
@ -428,6 +430,8 @@ func AuthorizeAccount(ctx context.Context, account, key string, opts ...AuthOpti
apiURI: b2resp.URI,
downloadURI: b2resp.DownloadURI,
minPartSize: b2resp.PartSize,
bucket: b2resp.Allowed.Bucket,
pfx: b2resp.Allowed.Prefix,
opts: b2opts,
}, nil
}
@ -614,6 +618,7 @@ func (b *Bucket) BaseURL() string {
func (b *B2) ListBuckets(ctx context.Context) ([]*Bucket, error) {
b2req := &b2types.ListBucketsRequest{
AccountID: b.accountID,
Bucket: b.bucket,
}
b2resp := &b2types.ListBucketsResponse{}
headers := map[string]string{
@ -967,6 +972,9 @@ func (b *Bucket) ListUnfinishedLargeFiles(ctx context.Context, count int, contin
// ListFileNames wraps b2_list_file_names.
func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, prefix, delimiter string) ([]*File, string, error) {
if prefix == "" {
prefix = b.b2.pfx
}
b2req := &b2types.ListFileNamesRequest{
Count: count,
Continuation: continuation,
@ -1007,6 +1015,9 @@ func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, pre
// ListFileVersions wraps b2_list_file_versions.
func (b *Bucket) ListFileVersions(ctx context.Context, count int, startName, startID, prefix, delimiter string) ([]*File, string, string, error) {
if prefix == "" {
prefix = b.b2.pfx
}
b2req := &b2types.ListFileVersionsRequest{
BucketID: b.ID,
Count: count,

8
vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go generated vendored
View file

@ -65,9 +65,7 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{})
if *c.pfx != "" {
opts = append(opts, b2.Prefix(*c.pfx))
}
for _, c := range caps {
opts = append(opts, b2.Capability(c))
}
opts = append(opts, b2.Capabilities(caps...))
client, err := b2.NewClient(ctx, id, key, b2.UserAgent("b2keys"))
if err != nil {
@ -86,10 +84,12 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{})
cr = bucket
}
if _, err := cr.CreateKey(ctx, name, opts...); err != nil {
b2key, err := cr.CreateKey(ctx, name, opts...)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return subcommands.ExitFailure
}
fmt.Printf("key=%s, secret=%s\n", b2key.ID(), b2key.Secret())
return subcommands.ExitSuccess
}

23
vendor/github.com/kurin/blazer/internal/b2types/b2types.go generated vendored
View file

@ -29,14 +29,20 @@ type ErrorMessage struct {
}
type AuthorizeAccountResponse struct {
AccountID string `json:"accountId"`
AuthToken string `json:"authorizationToken"`
URI string `json:"apiUrl"`
DownloadURI string `json:"downloadUrl"`
MinPartSize int `json:"minimumPartSize"`
PartSize int `json:"recommendedPartSize"`
AbsMinPartSize int `json:"absoluteMinimumPartSize"`
Capabilities []string `json:"capabilities"`
AccountID string `json:"accountId"`
AuthToken string `json:"authorizationToken"`
URI string `json:"apiUrl"`
DownloadURI string `json:"downloadUrl"`
MinPartSize int `json:"minimumPartSize"`
PartSize int `json:"recommendedPartSize"`
AbsMinPartSize int `json:"absoluteMinimumPartSize"`
Allowed Allowance `json:"allowed"`
}
type Allowance struct {
Capabilities []string `json:"capabilities"`
Bucket string `json:"bucketId"`
Prefix string `json:"namePrefix"`
}
type LifecycleRule struct {
@ -69,6 +75,7 @@ type DeleteBucketRequest struct {
type ListBucketsRequest struct {
AccountID string `json:"accountId"`
Bucket string `json:"bucketId,omitempty"`
}
type ListBucketsResponse struct {

35
vendor/github.com/kurin/blazer/internal/pyre/api.go generated vendored
View file

@ -255,9 +255,9 @@ func (s *Server) ListFileVersions(ctx context.Context, req *pb.ListFileVersionsR
return nil, nil
}
//type objTuple struct {
// name, version string
//}
type objTuple struct {
name, version string
}
type ListManager interface {
// NextN returns the next n objects, sorted by lexicographical order by name,
@ -276,6 +276,35 @@ type VersionedObject interface {
NextNVersions(begin string, n int) ([]string, error)
}
func getDirNames(lm ListManager, bucket, name, prefix, delim string, n int) ([]string, error) {
var sfx string
var out []string
for n > 0 {
vo, err := lm.NextN(bucket, name, prefix, sfx, 1)
if err != nil {
return nil, err
}
if len(vo) == 0 {
return out, nil
}
v := vo[0]
name = v.Name()
suffix := name[len(prefix):]
i := strings.Index(suffix, delim)
if i < 0 {
sfx = ""
out = append(out, name)
name += "\000"
n--
continue
}
sfx = v.Name()[:len(prefix)+i+1]
out = append(out, sfx)
n--
}
return out, nil
}
//func getNextObjects(lm ListManager, bucket, name, prefix, delimiter string, n int) ([]VersionedObject, error) {
// if delimiter == "" {
// return lm.NextN(bucket, name, prefix, "", n)

137
vendor/github.com/kurin/blazer/internal/pyre/api_test.go generated vendored Normal file
View file

@ -0,0 +1,137 @@
// Copyright 2018, Google
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pyre
import (
"reflect"
"sort"
"strings"
"sync"
"testing"
)
type testVersionedObject struct {
name string
versions []string
}
func (t testVersionedObject) Name() string { return t.name }
func (t testVersionedObject) NextNVersions(b string, n int) ([]string, error) {
var out []string
var seen bool
if b == "" {
seen = true
}
for _, v := range t.versions {
if b == v {
seen = true
}
if !seen {
continue
}
if len(out) >= n {
return out, nil
}
out = append(out, v)
}
return out, nil
}
type testListManager struct {
objs map[string][]string
m sync.Mutex
}
func (t *testListManager) NextN(b, fn, pfx, spfx string, n int) ([]VersionedObject, error) {
t.m.Lock()
defer t.m.Unlock()
var out []VersionedObject
var keys []string
for k := range t.objs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if k < fn {
continue
}
if !strings.HasPrefix(k, pfx) {
continue
}
if spfx != "" && strings.HasPrefix(k, spfx) {
continue
}
out = append(out, testVersionedObject{name: k, versions: t.objs[k]})
n--
if n <= 0 {
return out, nil
}
}
return out, nil
}
func TestGetDirNames(t *testing.T) {
table := []struct {
lm ListManager
name string
pfx string
delim string
num int
want []string
}{
{
lm: &testListManager{
objs: map[string][]string{
"/usr/local/etc/foo/bar": {"a"},
"/usr/local/etc/foo/baz": {"a"},
"/usr/local/etc/foo": {"a"},
"/usr/local/etc/fool": {"a"},
},
},
num: 2,
pfx: "/usr/local/etc/",
delim: "/",
want: []string{"/usr/local/etc/foo", "/usr/local/etc/foo/"},
},
{
lm: &testListManager{
objs: map[string][]string{
"/usr/local/etc/foo/bar": {"a"},
"/usr/local/etc/foo/baz": {"a"},
"/usr/local/etc/foo": {"a"},
"/usr/local/etc/fool": {"a"},
"/usr/local/etc/bar": {"a"},
},
},
num: 4,
pfx: "/usr/local/etc/",
delim: "/",
want: []string{"/usr/local/etc/bar", "/usr/local/etc/foo", "/usr/local/etc/foo/", "/usr/local/etc/fool"},
},
}
for _, e := range table {
got, err := getDirNames(e.lm, "", e.name, e.pfx, e.delim, e.num)
if err != nil {
t.Error(err)
continue
}
if !reflect.DeepEqual(got, e.want) {
t.Errorf("getDirNames(%v, %q, %q, %q, %d): got %v, want %v", e.lm, e.name, e.pfx, e.delim, e.num, got, e.want)
}
}
}