From f761068f4ed8064e5e13d41a71349b5dee491e33 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 9 Jun 2018 12:32:18 +0200 Subject: [PATCH] Update github.com/kurin/blazer --- Gopkg.lock | 4 +- vendor/github.com/kurin/blazer/README.md | 2 +- .../kurin/blazer/b2/integration_test.go | 30 +++++------ vendor/github.com/kurin/blazer/b2/iterator.go | 50 ++++++++++++++++--- vendor/github.com/kurin/blazer/base/base.go | 2 +- .../blazer/internal/bin/cleanup/cleanup.go | 4 +- .../blazer/x/consistent/consistent_test.go | 4 +- 7 files changed, 65 insertions(+), 31 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8ff9cc2bf..b2faafb4f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -94,8 +94,8 @@ [[projects]] name = "github.com/kurin/blazer" packages = ["b2","base","internal/b2assets","internal/b2types","internal/blog","x/window"] - revision = "3c18ed98a4120a440c8f45d8fbf41d414612a501" - version = "v0.4.2" + revision = "318e9768bf9a0fe52a64b9f8fe74f4f5caef6452" + version = "v0.4.4" [[projects]] name = "github.com/marstr/guid" diff --git a/vendor/github.com/kurin/blazer/README.md b/vendor/github.com/kurin/blazer/README.md index 35095536e..cd1a02768 100644 --- a/vendor/github.com/kurin/blazer/README.md +++ b/vendor/github.com/kurin/blazer/README.md @@ -97,7 +97,7 @@ func downloadFile(ctx context.Context, bucket *b2.Bucket, downloads int, src, ds ```go func printObjects(ctx context.Context, bucket *b2.Bucket) error { - iterator := bucket.List() + iterator := bucket.List(ctx) for iterator.Next() { fmt.Println(itrator.Object()) } diff --git a/vendor/github.com/kurin/blazer/b2/integration_test.go b/vendor/github.com/kurin/blazer/b2/integration_test.go index a428fce74..1f7acefd6 100644 --- a/vendor/github.com/kurin/blazer/b2/integration_test.go +++ b/vendor/github.com/kurin/blazer/b2/integration_test.go @@ -64,8 +64,8 @@ func TestReadWriteLive(t *testing.T) { t.Error(err) } - iter := bucket.List(ListHidden()) - for iter.Next(ctx) { + iter := bucket.List(ctx, ListHidden()) + for iter.Next() { if err := iter.Object().Delete(ctx); err != nil { t.Error(err) } @@ -168,7 +168,7 @@ func TestHideShowLive(t *testing.T) { t.Fatal(err) } - got, err := countObjects(ctx, bucket.List()) + got, err := countObjects(bucket.List(ctx)) if err != nil { t.Error(err) } @@ -186,7 +186,7 @@ func TestHideShowLive(t *testing.T) { t.Fatal(err) } - got, err = countObjects(ctx, bucket.List()) + got, err = countObjects(bucket.List(ctx)) if err != nil { t.Error(err) } @@ -200,7 +200,7 @@ func TestHideShowLive(t *testing.T) { } // count see the object again - got, err = countObjects(ctx, bucket.List()) + got, err = countObjects(bucket.List(ctx)) if err != nil { t.Error(err) } @@ -552,9 +552,9 @@ func TestListObjectsWithPrefix(t *testing.T) { } for _, entry := range table { - iter := bucket.List(entry.opts...) + iter := bucket.List(ctx, entry.opts...) var res []string - for iter.Next(ctx) { + for iter.Next() { o := iter.Object() attrs, err := o.Attrs(ctx) if err != nil { @@ -743,8 +743,8 @@ func TestAttrsNoRoundtrip(t *testing.T) { t.Fatal(err) } - iter := bucket.List() - iter.Next(ctx) + iter := bucket.List(ctx) + iter.Next() obj := iter.Object() var trips int @@ -852,8 +852,8 @@ func TestListUnfinishedLargeFiles(t *testing.T) { if _, err := io.Copy(w, io.LimitReader(zReader{}, 1e6)); err != nil { t.Fatal(err) } - iter := bucket.List(ListUnfinished()) - if !iter.Next(ctx) { + iter := bucket.List(ctx, ListUnfinished()) + if !iter.Next() { t.Errorf("ListUnfinishedLargeFiles: got none, want 1 (error %v)", iter.Err()) } } @@ -894,9 +894,9 @@ type object struct { err error } -func countObjects(ctx context.Context, iter *ObjectIterator) (int, error) { +func countObjects(iter *ObjectIterator) (int, error) { var got int - for iter.Next(ctx) { + for iter.Next() { got++ } return got, iter.Err() @@ -1004,8 +1004,8 @@ func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) { } f := func() { defer ccport.done() - iter := bucket.List(ListHidden()) - for iter.Next(ctx) { + iter := bucket.List(ctx, ListHidden()) + for iter.Next() { if err := iter.Object().Delete(ctx); err != nil { t.Error(err) } diff --git a/vendor/github.com/kurin/blazer/b2/iterator.go b/vendor/github.com/kurin/blazer/b2/iterator.go index 48927e05f..1679c81e3 100644 --- a/vendor/github.com/kurin/blazer/b2/iterator.go +++ b/vendor/github.com/kurin/blazer/b2/iterator.go @@ -22,9 +22,10 @@ import ( // List returns an iterator for selecting objects in a bucket. The default // behavior, with no options, is to list all currently un-hidden objects. -func (b *Bucket) List(opts ...ListOption) *ObjectIterator { +func (b *Bucket) List(ctx context.Context, opts ...ListOption) *ObjectIterator { o := &ObjectIterator{ bucket: b, + ctx: ctx, } for _, opt := range opts { opt(&o.opts) @@ -36,7 +37,7 @@ func (b *Bucket) List(opts ...ListOption) *ObjectIterator { // contents. // // It is intended to be called in a loop: -// for iter.Next(ctx) { +// for iter.Next() { // obj := iter.Object() // // act on obj // } @@ -45,6 +46,7 @@ func (b *Bucket) List(opts ...ListOption) *ObjectIterator { // } type ObjectIterator struct { bucket *Bucket + ctx context.Context final bool err error idx int @@ -58,7 +60,11 @@ type ObjectIterator struct { type lister func(context.Context, int, *Cursor) ([]*Object, *Cursor, error) -func (o *ObjectIterator) frame(ctx context.Context) error { +func (o *ObjectIterator) page(ctx context.Context) error { + if o.opts.locker != nil { + o.opts.locker.Lock() + defer o.opts.locker.Unlock() + } objs, c, err := o.l(ctx, o.count, o.c) if err != nil && err != io.EOF { if bNotExist.MatchString(err.Error()) { @@ -82,13 +88,18 @@ func (o *ObjectIterator) frame(ctx context.Context) error { // any calls to Object(). If Next returns true, then the next call to Object() // will be valid. Once Next returns false, it is important to check the return // value of Err(). -func (o *ObjectIterator) Next(ctx context.Context) bool { +func (o *ObjectIterator) Next() bool { o.init.Do(func() { - o.count = 1000 + o.count = o.opts.pageSize + if o.count < 0 || o.count > 1000 { + o.count = 1000 + } switch { case o.opts.unfinished: o.l = o.bucket.ListUnfinishedLargeFiles - o.count = 100 + if o.count > 100 { + o.count = 100 + } case o.opts.hidden: o.l = o.bucket.ListObjects default: @@ -102,16 +113,20 @@ func (o *ObjectIterator) Next(ctx context.Context) bool { if o.err != nil { return false } + if o.ctx.Err() != nil { + o.err = o.ctx.Err() + return false + } if o.idx >= len(o.objs) { if o.final { o.err = io.EOF return false } - if err := o.frame(ctx); err != nil { + if err := o.page(o.ctx); err != nil { o.err = err return false } - return o.Next(ctx) + return o.Next() } o.idx++ return true @@ -136,6 +151,8 @@ type objectIteratorOptions struct { unfinished bool prefix string delimiter string + pageSize int + locker sync.Locker } // A ListOption alters the default behavor of List. @@ -181,3 +198,20 @@ func ListDelimiter(delimiter string) ListOption { o.delimiter = delimiter } } + +// ListPageSize configures the iterator to request the given number of objects +// per network round-trip. The default (and maximum) is 1000 objects, except +// for unfinished large files, which is 100. +func ListPageSize(count int) ListOption { + return func(o *objectIteratorOptions) { + o.pageSize = count + } +} + +// ListLocker passes the iterator a lock which will be held during network +// round-trips. +func ListLocker(l sync.Locker) ListOption { + return func(o *objectIteratorOptions) { + o.locker = l + } +} diff --git a/vendor/github.com/kurin/blazer/base/base.go b/vendor/github.com/kurin/blazer/base/base.go index 3f610c00f..11cd0da5a 100644 --- a/vendor/github.com/kurin/blazer/base/base.go +++ b/vendor/github.com/kurin/blazer/base/base.go @@ -42,7 +42,7 @@ import ( const ( APIBase = "https://api.backblazeb2.com" - DefaultUserAgent = "blazer/0.4.2" + DefaultUserAgent = "blazer/0.4.4" ) type b2err struct { diff --git a/vendor/github.com/kurin/blazer/internal/bin/cleanup/cleanup.go b/vendor/github.com/kurin/blazer/internal/bin/cleanup/cleanup.go index 47f2fce5f..2f90e95b5 100644 --- a/vendor/github.com/kurin/blazer/internal/bin/cleanup/cleanup.go +++ b/vendor/github.com/kurin/blazer/internal/bin/cleanup/cleanup.go @@ -61,8 +61,8 @@ func killBucket(ctx context.Context, client *b2.Client, name string) error { return err } defer bucket.Delete(ctx) - iter := bucket.List(b2.ListHidden()) - for iter.Next(ctx) { + iter := bucket.List(ctx, b2.ListHidden()) + for iter.Next() { if err := iter.Object().Delete(ctx); err != nil { fmt.Println(err) } diff --git a/vendor/github.com/kurin/blazer/x/consistent/consistent_test.go b/vendor/github.com/kurin/blazer/x/consistent/consistent_test.go index b9a50b587..0a94df72a 100644 --- a/vendor/github.com/kurin/blazer/x/consistent/consistent_test.go +++ b/vendor/github.com/kurin/blazer/x/consistent/consistent_test.go @@ -141,8 +141,8 @@ func startLiveTest(ctx context.Context, t *testing.T) (*b2.Bucket, func()) { return nil, nil } f := func() { - iter := bucket.List(b2.ListHidden()) - for iter.Next(ctx) { + iter := bucket.List(ctx, b2.ListHidden()) + for iter.Next() { if err := iter.Object().Delete(ctx); err != nil { t.Error(err) }