From 47186806cfa0c6a96afe7e32756cd0e6e02a5cad Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 10 May 2015 10:20:58 -0500 Subject: [PATCH 01/13] Add Amazon S3 backend. --- backend/s3/s3.go | 200 +++++++++++++++++++++++++++++++++++++++++++++ cmd/restic/main.go | 6 ++ 2 files changed, 206 insertions(+) create mode 100644 backend/s3/s3.go diff --git a/backend/s3/s3.go b/backend/s3/s3.go new file mode 100644 index 000000000..47ac21871 --- /dev/null +++ b/backend/s3/s3.go @@ -0,0 +1,200 @@ +package s3 + +import ( + "bytes" + "errors" + "io" + "strings" + "sync" + + "github.com/mitchellh/goamz/aws" + as3 "github.com/mitchellh/goamz/s3" + + "github.com/restic/restic/backend" +) + +var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") + +func s3path(t backend.Type, name string) string { + if t == backend.Config { + return string(t) + } + return string(t) + "/" + name +} + +type S3DB struct { + bucket *as3.Bucket + mput sync.Mutex + path string +} + +// Open opens the local backend at bucket and region. +func Open(regionname, bucketname string) (*S3DB, error) { + auth, err := aws.EnvAuth() + if err != nil { + return nil, err + } + + client := as3.New(auth, aws.Regions[regionname]) + + return &S3DB{bucket: client.Bucket(bucketname), path: bucketname}, nil +} + +// Location returns this backend's location (the directory name). +func (b *S3DB) Location() string { + return b.path +} + +type s3Blob struct { + b *S3DB + buf *bytes.Buffer + final bool +} + +func (bb *s3Blob) Write(p []byte) (int, error) { + if bb.final { + return 0, errors.New("blob already closed") + } + + n, err := bb.buf.Write(p) + return n, err +} + +func (bb *s3Blob) Read(p []byte) (int, error) { + return bb.buf.Read(p) +} + +func (bb *s3Blob) Close() error { + bb.final = true + bb.buf.Reset() + return nil +} + +func (bb *s3Blob) Size() uint { + return uint(bb.buf.Len()) +} + +func (bb *s3Blob) Finalize(t backend.Type, name string) error { + if bb.final { + return errors.New("Already finalized") + } + + bb.final = true + + path := s3path(t, name) + bb.b.mput.Lock() + err := bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") + bb.b.mput.Unlock() + bb.buf.Reset() + return err +} + +// Create creates a new Blob. The data is available only after Finalize() +// has been called on the returned Blob. +func (b *S3DB) Create() (backend.Blob, error) { + blob := s3Blob{ + b: b, + buf: &bytes.Buffer{}, + } + + return &blob, nil +} + +func (b *S3DB) get(t backend.Type, name string) (*s3Blob, error) { + blob := &s3Blob{ + b: b, + } + + path := s3path(t, name) + data, err := b.bucket.Get(path) + blob.buf = bytes.NewBuffer(data) + return blob, err +} + +// Get returns a reader that yields the content stored under the given +// name. The reader should be closed after draining it. +func (b *S3DB) Get(t backend.Type, name string) (io.ReadCloser, error) { + return b.get(t, name) +} + +// GetReader returns an io.ReadCloser for the Blob with the given name of +// type t at offset and length. If length is 0, the reader reads until EOF. +func (b *S3DB) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { + blob, err := b.get(t, name) + if err != nil { + return nil, err + } + + blob.buf.Next(int(offset)) + + if length == 0 { + return blob, nil + } + + return backend.LimitReadCloser(blob, int64(length)), nil +} + +// Test returns true if a blob of the given type and name exists in the backend. +func (b *S3DB) Test(t backend.Type, name string) (bool, error) { + found := false + path := s3path(t, name) + key, err := b.bucket.GetKey(path) + if err == nil && key.Key == path { + found = true + } + + // If error, then not found + return found, nil +} + +// Remove removes the blob with the given name and type. +func (b *S3DB) Remove(t backend.Type, name string) error { + path := s3path(t, name) + return b.bucket.Del(path) +} + +// List returns a channel that yields all names of blobs of type t. A +// goroutine is started for this. If the channel done is closed, sending +// stops. +func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string { + ch := make(chan string) + + prefix := string(t) + "/" + + listresp, err := b.bucket.List(prefix, "/", "", 10000) + + if err != nil { + close(ch) + return ch + } + + matches := make([]string, len(listresp.Contents)) + for idx, key := range listresp.Contents { + matches[idx] = strings.TrimPrefix(key.Key, prefix) + } + + go func() { + defer close(ch) + for _, m := range matches { + if m == "" { + continue + } + + select { + case ch <- m: + case <-done: + return + } + } + }() + + return ch +} + +// Delete removes the repository and all files. +func (b *S3DB) Delete() error { + return b.bucket.DelBucket() +} + +// Close does nothing +func (b *S3DB) Close() error { return nil } diff --git a/cmd/restic/main.go b/cmd/restic/main.go index b34755c84..86baa34ab 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -12,6 +12,7 @@ import ( "github.com/jessevdk/go-flags" "github.com/restic/restic/backend" "github.com/restic/restic/backend/local" + "github.com/restic/restic/backend/s3" "github.com/restic/restic/backend/sftp" "github.com/restic/restic/debug" "github.com/restic/restic/repository" @@ -119,6 +120,7 @@ func (cmd CmdInit) Execute(args []string) error { // Open the backend specified by URI. // Valid formats are: // * /foo/bar -> local repository at /foo/bar +// * s3://region/bucket -> amazon s3 bucket // * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar // * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup func open(u string) (backend.Backend, error) { @@ -129,6 +131,8 @@ func open(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Open(url.Path) + } else if url.Scheme == "s3" { + return s3.Open(url.Host, url.Path[1:]) } args := []string{url.Host} @@ -150,6 +154,8 @@ func create(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Create(url.Path) + } else if url.Scheme == "s3" { + return s3.Open(url.Host, url.Path[1:]) } args := []string{url.Host} From bfe221e71ccdc869049d15563e67c247810ef27d Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Wed, 13 May 2015 12:48:52 -0500 Subject: [PATCH 02/13] Handle long list responses from s3. Also change naming of backend struct. --- backend/s3/s3.go | 55 ++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 47ac21871..48f7beaa6 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -8,12 +8,12 @@ import ( "sync" "github.com/mitchellh/goamz/aws" - as3 "github.com/mitchellh/goamz/s3" + "github.com/mitchellh/goamz/s3" "github.com/restic/restic/backend" ) -var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") +const maxKeysInList = 1000 func s3path(t backend.Type, name string) string { if t == backend.Config { @@ -22,31 +22,31 @@ func s3path(t backend.Type, name string) string { return string(t) + "/" + name } -type S3DB struct { - bucket *as3.Bucket +type S3 struct { + bucket *s3.Bucket mput sync.Mutex path string } -// Open opens the local backend at bucket and region. -func Open(regionname, bucketname string) (*S3DB, error) { +// Open opens the s3 backend at bucket and region. +func Open(regionname, bucketname string) (*S3, error) { auth, err := aws.EnvAuth() if err != nil { return nil, err } - client := as3.New(auth, aws.Regions[regionname]) + client := s3.New(auth, aws.Regions[regionname]) - return &S3DB{bucket: client.Bucket(bucketname), path: bucketname}, nil + return &S3{bucket: client.Bucket(bucketname), path: bucketname}, nil } -// Location returns this backend's location (the directory name). -func (b *S3DB) Location() string { +// Location returns this backend's location (the bucket name). +func (b *S3) Location() string { return b.path } type s3Blob struct { - b *S3DB + b *S3 buf *bytes.Buffer final bool } @@ -91,7 +91,7 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { // Create creates a new Blob. The data is available only after Finalize() // has been called on the returned Blob. -func (b *S3DB) Create() (backend.Blob, error) { +func (b *S3) Create() (backend.Blob, error) { blob := s3Blob{ b: b, buf: &bytes.Buffer{}, @@ -100,7 +100,7 @@ func (b *S3DB) Create() (backend.Blob, error) { return &blob, nil } -func (b *S3DB) get(t backend.Type, name string) (*s3Blob, error) { +func (b *S3) get(t backend.Type, name string) (*s3Blob, error) { blob := &s3Blob{ b: b, } @@ -113,13 +113,13 @@ func (b *S3DB) get(t backend.Type, name string) (*s3Blob, error) { // Get returns a reader that yields the content stored under the given // name. The reader should be closed after draining it. -func (b *S3DB) Get(t backend.Type, name string) (io.ReadCloser, error) { +func (b *S3) Get(t backend.Type, name string) (io.ReadCloser, error) { return b.get(t, name) } // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. -func (b *S3DB) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { +func (b *S3) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { blob, err := b.get(t, name) if err != nil { return nil, err @@ -135,7 +135,7 @@ func (b *S3DB) GetReader(t backend.Type, name string, offset, length uint) (io.R } // Test returns true if a blob of the given type and name exists in the backend. -func (b *S3DB) Test(t backend.Type, name string) (bool, error) { +func (b *S3) Test(t backend.Type, name string) (bool, error) { found := false path := s3path(t, name) key, err := b.bucket.GetKey(path) @@ -148,7 +148,7 @@ func (b *S3DB) Test(t backend.Type, name string) (bool, error) { } // Remove removes the blob with the given name and type. -func (b *S3DB) Remove(t backend.Type, name string) error { +func (b *S3) Remove(t backend.Type, name string) error { path := s3path(t, name) return b.bucket.Del(path) } @@ -156,12 +156,12 @@ func (b *S3DB) Remove(t backend.Type, name string) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string { +func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { ch := make(chan string) prefix := string(t) + "/" - listresp, err := b.bucket.List(prefix, "/", "", 10000) + listresp, err := b.bucket.List(prefix, "/", "", maxKeysInList) if err != nil { close(ch) @@ -173,6 +173,19 @@ func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string { matches[idx] = strings.TrimPrefix(key.Key, prefix) } + // Continue making requests to get full list. + for listresp.IsTruncated { + listresp, err = b.bucket.List(prefix, "/", listresp.NextMarker, maxKeysInList) + if err != nil { + close(ch) + return ch + } + + for _, key := range listresp.Contents { + matches = append(matches, strings.TrimPrefix(key.Key, prefix)) + } + } + go func() { defer close(ch) for _, m := range matches { @@ -192,9 +205,9 @@ func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string { } // Delete removes the repository and all files. -func (b *S3DB) Delete() error { +func (b *S3) Delete() error { return b.bucket.DelBucket() } // Close does nothing -func (b *S3DB) Close() error { return nil } +func (b *S3) Close() error { return nil } From dc6d92a0761338eea344284e2493a8111fb3c50e Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Wed, 13 May 2015 13:11:07 -0500 Subject: [PATCH 03/13] Add Godeps for s3 backend. --- Godeps/Godeps.json | 12 + .../github.com/mitchellh/goamz/aws/attempt.go | 74 ++ .../mitchellh/goamz/aws/attempt_test.go | 57 ++ .../src/github.com/mitchellh/goamz/aws/aws.go | 445 +++++++++ .../mitchellh/goamz/aws/aws_test.go | 203 ++++ .../github.com/mitchellh/goamz/aws/client.go | 125 +++ .../mitchellh/goamz/aws/client_test.go | 121 +++ .../mitchellh/goamz/s3/export_test.go | 27 + .../github.com/mitchellh/goamz/s3/multi.go | 409 ++++++++ .../mitchellh/goamz/s3/multi_test.go | 370 ++++++++ .../mitchellh/goamz/s3/responses_test.go | 241 +++++ .../src/github.com/mitchellh/goamz/s3/s3.go | 893 ++++++++++++++++++ .../github.com/mitchellh/goamz/s3/s3_test.go | 435 +++++++++ .../github.com/mitchellh/goamz/s3/s3i_test.go | 616 ++++++++++++ .../github.com/mitchellh/goamz/s3/s3t_test.go | 79 ++ .../mitchellh/goamz/s3/s3test/server.go | 666 +++++++++++++ .../src/github.com/mitchellh/goamz/s3/sign.go | 126 +++ .../mitchellh/goamz/s3/sign_test.go | 194 ++++ .../src/github.com/vaughan0/go-ini/LICENSE | 14 + .../src/github.com/vaughan0/go-ini/README.md | 70 ++ .../src/github.com/vaughan0/go-ini/ini.go | 123 +++ .../vaughan0/go-ini/ini_linux_test.go | 43 + .../github.com/vaughan0/go-ini/ini_test.go | 89 ++ .../src/github.com/vaughan0/go-ini/test.ini | 2 + 24 files changed, 5434 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go create mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 32a7cad3c..761916140 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -18,10 +18,22 @@ "ImportPath": "github.com/kr/fs", "Rev": "2788f0dbd16903de03cb8186e5c7d97b69ad387b" }, + { + "ImportPath": "github.com/mitchellh/goamz/aws", + "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" + }, + { + "ImportPath": "github.com/mitchellh/goamz/s3", + "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" + }, { "ImportPath": "github.com/pkg/sftp", "Rev": "506297c9013d2893d5c5daaa9155e7333a1c58de" }, + { + "ImportPath": "github.com/vaughan0/go-ini", + "Rev": "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1" + }, { "ImportPath": "golang.org/x/crypto/pbkdf2", "Rev": "24ffb5feb3312a39054178a4b0a4554fc2201248" diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go new file mode 100644 index 000000000..c0654f5d8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go @@ -0,0 +1,74 @@ +package aws + +import ( + "time" +) + +// AttemptStrategy represents a strategy for waiting for an action +// to complete successfully. This is an internal type used by the +// implementation of other goamz packages. +type AttemptStrategy struct { + Total time.Duration // total duration of attempt. + Delay time.Duration // interval between each try in the burst. + Min int // minimum number of retries; overrides Total +} + +type Attempt struct { + strategy AttemptStrategy + last time.Time + end time.Time + force bool + count int +} + +// Start begins a new sequence of attempts for the given strategy. +func (s AttemptStrategy) Start() *Attempt { + now := time.Now() + return &Attempt{ + strategy: s, + last: now, + end: now.Add(s.Total), + force: true, + } +} + +// Next waits until it is time to perform the next attempt or returns +// false if it is time to stop trying. +func (a *Attempt) Next() bool { + now := time.Now() + sleep := a.nextSleep(now) + if !a.force && !now.Add(sleep).Before(a.end) && a.strategy.Min <= a.count { + return false + } + a.force = false + if sleep > 0 && a.count > 0 { + time.Sleep(sleep) + now = time.Now() + } + a.count++ + a.last = now + return true +} + +func (a *Attempt) nextSleep(now time.Time) time.Duration { + sleep := a.strategy.Delay - now.Sub(a.last) + if sleep < 0 { + return 0 + } + return sleep +} + +// HasNext returns whether another attempt will be made if the current +// one fails. If it returns true, the following call to Next is +// guaranteed to return true. +func (a *Attempt) HasNext() bool { + if a.force || a.strategy.Min > a.count { + return true + } + now := time.Now() + if now.Add(a.nextSleep(now)).Before(a.end) { + a.force = true + return true + } + return false +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go new file mode 100644 index 000000000..1fda5bf3c --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go @@ -0,0 +1,57 @@ +package aws_test + +import ( + "github.com/mitchellh/goamz/aws" + . "github.com/motain/gocheck" + "time" +) + +func (S) TestAttemptTiming(c *C) { + testAttempt := aws.AttemptStrategy{ + Total: 0.25e9, + Delay: 0.1e9, + } + want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9} + got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing + t0 := time.Now() + for a := testAttempt.Start(); a.Next(); { + got = append(got, time.Now().Sub(t0)) + } + got = append(got, time.Now().Sub(t0)) + c.Assert(got, HasLen, len(want)) + const margin = 0.01e9 + for i, got := range want { + lo := want[i] - margin + hi := want[i] + margin + if got < lo || got > hi { + c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds()) + } + } +} + +func (S) TestAttemptNextHasNext(c *C) { + a := aws.AttemptStrategy{}.Start() + c.Assert(a.Next(), Equals, true) + c.Assert(a.Next(), Equals, false) + + a = aws.AttemptStrategy{}.Start() + c.Assert(a.Next(), Equals, true) + c.Assert(a.HasNext(), Equals, false) + c.Assert(a.Next(), Equals, false) + + a = aws.AttemptStrategy{Total: 2e8}.Start() + c.Assert(a.Next(), Equals, true) + c.Assert(a.HasNext(), Equals, true) + time.Sleep(2e8) + c.Assert(a.HasNext(), Equals, true) + c.Assert(a.Next(), Equals, true) + c.Assert(a.Next(), Equals, false) + + a = aws.AttemptStrategy{Total: 1e8, Min: 2}.Start() + time.Sleep(1e8) + c.Assert(a.Next(), Equals, true) + c.Assert(a.HasNext(), Equals, true) + c.Assert(a.Next(), Equals, true) + c.Assert(a.HasNext(), Equals, false) + c.Assert(a.Next(), Equals, false) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go new file mode 100644 index 000000000..cfc42c03a --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go @@ -0,0 +1,445 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// +// Written by Gustavo Niemeyer +// +package aws + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/vaughan0/go-ini" +) + +// Region defines the URLs where AWS services may be accessed. +// +// See http://goo.gl/d8BP1 for more details. +type Region struct { + Name string // the canonical name of this region. + EC2Endpoint string + S3Endpoint string + S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name. + S3LocationConstraint bool // true if this region requires a LocationConstraint declaration. + S3LowercaseBucket bool // true if the region requires bucket names to be lower case. + SDBEndpoint string + SNSEndpoint string + SQSEndpoint string + IAMEndpoint string + ELBEndpoint string + AutoScalingEndpoint string + RdsEndpoint string + Route53Endpoint string +} + +var USGovWest = Region{ + "us-gov-west-1", + "https://ec2.us-gov-west-1.amazonaws.com", + "https://s3-fips-us-gov-west-1.amazonaws.com", + "", + true, + true, + "", + "https://sns.us-gov-west-1.amazonaws.com", + "https://sqs.us-gov-west-1.amazonaws.com", + "https://iam.us-gov.amazonaws.com", + "https://elasticloadbalancing.us-gov-west-1.amazonaws.com", + "https://autoscaling.us-gov-west-1.amazonaws.com", + "https://rds.us-gov-west-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var USEast = Region{ + "us-east-1", + "https://ec2.us-east-1.amazonaws.com", + "https://s3.amazonaws.com", + "", + false, + false, + "https://sdb.amazonaws.com", + "https://sns.us-east-1.amazonaws.com", + "https://sqs.us-east-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-east-1.amazonaws.com", + "https://autoscaling.us-east-1.amazonaws.com", + "https://rds.us-east-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var USWest = Region{ + "us-west-1", + "https://ec2.us-west-1.amazonaws.com", + "https://s3-us-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-1.amazonaws.com", + "https://sns.us-west-1.amazonaws.com", + "https://sqs.us-west-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-west-1.amazonaws.com", + "https://autoscaling.us-west-1.amazonaws.com", + "https://rds.us-west-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var USWest2 = Region{ + "us-west-2", + "https://ec2.us-west-2.amazonaws.com", + "https://s3-us-west-2.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-2.amazonaws.com", + "https://sns.us-west-2.amazonaws.com", + "https://sqs.us-west-2.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-west-2.amazonaws.com", + "https://autoscaling.us-west-2.amazonaws.com", + "https://rds.us-west-2.amazonaws.com", + "https://route53.amazonaws.com", +} + +var EUWest = Region{ + "eu-west-1", + "https://ec2.eu-west-1.amazonaws.com", + "https://s3-eu-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.eu-west-1.amazonaws.com", + "https://sns.eu-west-1.amazonaws.com", + "https://sqs.eu-west-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.eu-west-1.amazonaws.com", + "https://autoscaling.eu-west-1.amazonaws.com", + "https://rds.eu-west-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var EUCentral = Region{ + "eu-central-1", + "https://ec2.eu-central-1.amazonaws.com", + "https://s3-eu-central-1.amazonaws.com", + "", + true, + true, + "", + "https://sns.eu-central-1.amazonaws.com", + "https://sqs.eu-central-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.eu-central-1.amazonaws.com", + "https://autoscaling.eu-central-1.amazonaws.com", + "https://rds.eu-central-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var APSoutheast = Region{ + "ap-southeast-1", + "https://ec2.ap-southeast-1.amazonaws.com", + "https://s3-ap-southeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-1.amazonaws.com", + "https://sns.ap-southeast-1.amazonaws.com", + "https://sqs.ap-southeast-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-southeast-1.amazonaws.com", + "https://autoscaling.ap-southeast-1.amazonaws.com", + "https://rds.ap-southeast-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var APSoutheast2 = Region{ + "ap-southeast-2", + "https://ec2.ap-southeast-2.amazonaws.com", + "https://s3-ap-southeast-2.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-2.amazonaws.com", + "https://sns.ap-southeast-2.amazonaws.com", + "https://sqs.ap-southeast-2.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-southeast-2.amazonaws.com", + "https://autoscaling.ap-southeast-2.amazonaws.com", + "https://rds.ap-southeast-2.amazonaws.com", + "https://route53.amazonaws.com", +} + +var APNortheast = Region{ + "ap-northeast-1", + "https://ec2.ap-northeast-1.amazonaws.com", + "https://s3-ap-northeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-northeast-1.amazonaws.com", + "https://sns.ap-northeast-1.amazonaws.com", + "https://sqs.ap-northeast-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-northeast-1.amazonaws.com", + "https://autoscaling.ap-northeast-1.amazonaws.com", + "https://rds.ap-northeast-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var SAEast = Region{ + "sa-east-1", + "https://ec2.sa-east-1.amazonaws.com", + "https://s3-sa-east-1.amazonaws.com", + "", + true, + true, + "https://sdb.sa-east-1.amazonaws.com", + "https://sns.sa-east-1.amazonaws.com", + "https://sqs.sa-east-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.sa-east-1.amazonaws.com", + "https://autoscaling.sa-east-1.amazonaws.com", + "https://rds.sa-east-1.amazonaws.com", + "https://route53.amazonaws.com", +} + +var CNNorth = Region{ + "cn-north-1", + "https://ec2.cn-north-1.amazonaws.com.cn", + "https://s3.cn-north-1.amazonaws.com.cn", + "", + true, + true, + "", + "https://sns.cn-north-1.amazonaws.com.cn", + "https://sqs.cn-north-1.amazonaws.com.cn", + "https://iam.cn-north-1.amazonaws.com.cn", + "https://elasticloadbalancing.cn-north-1.amazonaws.com.cn", + "https://autoscaling.cn-north-1.amazonaws.com.cn", + "https://rds.cn-north-1.amazonaws.com.cn", + "https://route53.amazonaws.com", +} + +var Regions = map[string]Region{ + APNortheast.Name: APNortheast, + APSoutheast.Name: APSoutheast, + APSoutheast2.Name: APSoutheast2, + EUWest.Name: EUWest, + EUCentral.Name: EUCentral, + USEast.Name: USEast, + USWest.Name: USWest, + USWest2.Name: USWest2, + SAEast.Name: SAEast, + USGovWest.Name: USGovWest, + CNNorth.Name: CNNorth, +} + +type Auth struct { + AccessKey, SecretKey, Token string +} + +var unreserved = make([]bool, 128) +var hex = "0123456789ABCDEF" + +func init() { + // RFC3986 + u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~" + for _, c := range u { + unreserved[c] = true + } +} + +type credentials struct { + Code string + LastUpdated string + Type string + AccessKeyId string + SecretAccessKey string + Token string + Expiration string +} + +// GetMetaData retrieves instance metadata about the current machine. +// +// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html for more details. +func GetMetaData(path string) (contents []byte, err error) { + url := "http://169.254.169.254/latest/meta-data/" + path + + resp, err := RetryingClient.Get(url) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + err = fmt.Errorf("Code %d returned for url %s", resp.StatusCode, url) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + return []byte(body), err +} + +func getInstanceCredentials() (cred credentials, err error) { + credentialPath := "iam/security-credentials/" + + // Get the instance role + role, err := GetMetaData(credentialPath) + if err != nil { + return + } + + // Get the instance role credentials + credentialJSON, err := GetMetaData(credentialPath + string(role)) + if err != nil { + return + } + + err = json.Unmarshal([]byte(credentialJSON), &cred) + return +} + +// GetAuth creates an Auth based on either passed in credentials, +// environment information or instance based role credentials. +func GetAuth(accessKey string, secretKey string) (auth Auth, err error) { + // First try passed in credentials + if accessKey != "" && secretKey != "" { + return Auth{accessKey, secretKey, ""}, nil + } + + // Next try to get auth from the environment + auth, err = SharedAuth() + if err == nil { + // Found auth, return + return + } + + // Next try to get auth from the environment + auth, err = EnvAuth() + if err == nil { + // Found auth, return + return + } + + // Next try getting auth from the instance role + cred, err := getInstanceCredentials() + if err == nil { + // Found auth, return + auth.AccessKey = cred.AccessKeyId + auth.SecretKey = cred.SecretAccessKey + auth.Token = cred.Token + return + } + err = errors.New("No valid AWS authentication found") + return +} + +// SharedAuth creates an Auth based on shared credentials stored in +// $HOME/.aws/credentials. The AWS_PROFILE environment variables is used to +// select the profile. +func SharedAuth() (auth Auth, err error) { + var profileName = os.Getenv("AWS_PROFILE") + + if profileName == "" { + profileName = "default" + } + + var credentialsFile = os.Getenv("AWS_CREDENTIAL_FILE") + if credentialsFile == "" { + var homeDir = os.Getenv("HOME") + if homeDir == "" { + err = errors.New("Could not get HOME") + return + } + credentialsFile = homeDir + "/.aws/credentials" + } + + file, err := ini.LoadFile(credentialsFile) + if err != nil { + err = errors.New("Couldn't parse AWS credentials file") + return + } + + var profile = file[profileName] + if profile == nil { + err = errors.New("Couldn't find profile in AWS credentials file") + return + } + + auth.AccessKey = profile["aws_access_key_id"] + auth.SecretKey = profile["aws_secret_access_key"] + + if auth.AccessKey == "" { + err = errors.New("AWS_ACCESS_KEY_ID not found in environment in credentials file") + } + if auth.SecretKey == "" { + err = errors.New("AWS_SECRET_ACCESS_KEY not found in credentials file") + } + return +} + +// EnvAuth creates an Auth based on environment information. +// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment +// For accounts that require a security token, it is read from AWS_SECURITY_TOKEN +// variables are used. +func EnvAuth() (auth Auth, err error) { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + if auth.AccessKey == "" { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY") + } + + auth.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + if auth.SecretKey == "" { + auth.SecretKey = os.Getenv("AWS_SECRET_KEY") + } + + auth.Token = os.Getenv("AWS_SECURITY_TOKEN") + + if auth.AccessKey == "" { + err = errors.New("AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") + } + if auth.SecretKey == "" { + err = errors.New("AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") + } + return +} + +// Encode takes a string and URI-encodes it in a way suitable +// to be used in AWS signatures. +func Encode(s string) string { + encode := false + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + encode = true + break + } + } + if !encode { + return s + } + e := make([]byte, len(s)*3) + ei := 0 + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + e[ei] = '%' + e[ei+1] = hex[c>>4] + e[ei+2] = hex[c&0xF] + ei += 3 + } else { + e[ei] = c + ei += 1 + } + } + return string(e[:ei]) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go new file mode 100644 index 000000000..78cbbaf03 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go @@ -0,0 +1,203 @@ +package aws_test + +import ( + "github.com/mitchellh/goamz/aws" + . "github.com/motain/gocheck" + "io/ioutil" + "os" + "strings" + "testing" +) + +func Test(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&S{}) + +type S struct { + environ []string +} + +func (s *S) SetUpSuite(c *C) { + s.environ = os.Environ() +} + +func (s *S) TearDownTest(c *C) { + os.Clearenv() + for _, kv := range s.environ { + l := strings.SplitN(kv, "=", 2) + os.Setenv(l[0], l[1]) + } +} + +func (s *S) TestSharedAuthNoHome(c *C) { + os.Clearenv() + os.Setenv("AWS_PROFILE", "foo") + _, err := aws.SharedAuth() + c.Assert(err, ErrorMatches, "Could not get HOME") +} + +func (s *S) TestSharedAuthNoCredentialsFile(c *C) { + os.Clearenv() + os.Setenv("AWS_PROFILE", "foo") + os.Setenv("HOME", "/tmp") + _, err := aws.SharedAuth() + c.Assert(err, ErrorMatches, "Couldn't parse AWS credentials file") +} + +func (s *S) TestSharedAuthNoProfileInFile(c *C) { + os.Clearenv() + os.Setenv("AWS_PROFILE", "foo") + + d, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(d) + + err = os.Mkdir(d+"/.aws", 0755) + if err != nil { + panic(err) + } + + ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\n"), 0644) + os.Setenv("HOME", d) + + _, err = aws.SharedAuth() + c.Assert(err, ErrorMatches, "Couldn't find profile in AWS credentials file") +} + +func (s *S) TestSharedAuthNoKeysInProfile(c *C) { + os.Clearenv() + os.Setenv("AWS_PROFILE", "bar") + + d, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(d) + + err = os.Mkdir(d+"/.aws", 0755) + if err != nil { + panic(err) + } + + ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\nawsaccesskeyid = AK.."), 0644) + os.Setenv("HOME", d) + + _, err = aws.SharedAuth() + c.Assert(err, ErrorMatches, "AWS_SECRET_ACCESS_KEY not found in credentials file") +} + +func (s *S) TestSharedAuthDefaultCredentials(c *C) { + os.Clearenv() + + d, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(d) + + err = os.Mkdir(d+"/.aws", 0755) + if err != nil { + panic(err) + } + + ioutil.WriteFile(d+"/.aws/credentials", []byte("[default]\naws_access_key_id = access\naws_secret_access_key = secret\n"), 0644) + os.Setenv("HOME", d) + + auth, err := aws.SharedAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestSharedAuth(c *C) { + os.Clearenv() + os.Setenv("AWS_PROFILE", "bar") + + d, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(d) + + err = os.Mkdir(d+"/.aws", 0755) + if err != nil { + panic(err) + } + + ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\naws_access_key_id = access\naws_secret_access_key = secret\n"), 0644) + os.Setenv("HOME", d) + + auth, err := aws.SharedAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEnvAuthNoSecret(c *C) { + os.Clearenv() + _, err := aws.EnvAuth() + c.Assert(err, ErrorMatches, "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") +} + +func (s *S) TestEnvAuthNoAccess(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "foo") + _, err := aws.EnvAuth() + c.Assert(err, ErrorMatches, "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") +} + +func (s *S) TestEnvAuth(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY_ID", "access") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEnvAuthWithToken(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY_ID", "access") + os.Setenv("AWS_SECURITY_TOKEN", "token") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access", Token: "token"}) +} + +func (s *S) TestEnvAuthAlt(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY", "access") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestGetAuthStatic(c *C) { + auth, err := aws.GetAuth("access", "secret") + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestGetAuthEnv(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY_ID", "access") + auth, err := aws.GetAuth("", "") + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEncode(c *C) { + c.Assert(aws.Encode("foo"), Equals, "foo") + c.Assert(aws.Encode("/"), Equals, "%2F") +} + +func (s *S) TestRegionsAreNamed(c *C) { + for n, r := range aws.Regions { + c.Assert(n, Equals, r.Name) + } +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go new file mode 100644 index 000000000..ee53238f7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go @@ -0,0 +1,125 @@ +package aws + +import ( + "math" + "net" + "net/http" + "time" +) + +type RetryableFunc func(*http.Request, *http.Response, error) bool +type WaitFunc func(try int) +type DeadlineFunc func() time.Time + +type ResilientTransport struct { + // Timeout is the maximum amount of time a dial will wait for + // a connect to complete. + // + // The default is no timeout. + // + // With or without a timeout, the operating system may impose + // its own earlier timeout. For instance, TCP timeouts are + // often around 3 minutes. + DialTimeout time.Duration + + // MaxTries, if non-zero, specifies the number of times we will retry on + // failure. Retries are only attempted for temporary network errors or known + // safe failures. + MaxTries int + Deadline DeadlineFunc + ShouldRetry RetryableFunc + Wait WaitFunc + transport *http.Transport +} + +// Convenience method for creating an http client +func NewClient(rt *ResilientTransport) *http.Client { + rt.transport = &http.Transport{ + Dial: func(netw, addr string) (net.Conn, error) { + c, err := net.DialTimeout(netw, addr, rt.DialTimeout) + if err != nil { + return nil, err + } + c.SetDeadline(rt.Deadline()) + return c, nil + }, + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + } + // TODO: Would be nice is ResilientTransport allowed clients to initialize + // with http.Transport attributes. + return &http.Client{ + Transport: rt, + } +} + +var retryingTransport = &ResilientTransport{ + Deadline: func() time.Time { + return time.Now().Add(5 * time.Second) + }, + DialTimeout: 10 * time.Second, + MaxTries: 3, + ShouldRetry: awsRetry, + Wait: ExpBackoff, +} + +// Exported default client +var RetryingClient = NewClient(retryingTransport) + +func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.tries(req) +} + +// Retry a request a maximum of t.MaxTries times. +// We'll only retry if the proper criteria are met. +// If a wait function is specified, wait that amount of time +// In between requests. +func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) { + for try := 0; try < t.MaxTries; try += 1 { + res, err = t.transport.RoundTrip(req) + + if !t.ShouldRetry(req, res, err) { + break + } + if res != nil { + res.Body.Close() + } + if t.Wait != nil { + t.Wait(try) + } + } + + return +} + +func ExpBackoff(try int) { + time.Sleep(100 * time.Millisecond * + time.Duration(math.Exp2(float64(try)))) +} + +func LinearBackoff(try int) { + time.Sleep(time.Duration(try*100) * time.Millisecond) +} + +// Decide if we should retry a request. +// In general, the criteria for retrying a request is described here +// http://docs.aws.amazon.com/general/latest/gr/api-retries.html +func awsRetry(req *http.Request, res *http.Response, err error) bool { + retry := false + + // Retry if there's a temporary network error. + if neterr, ok := err.(net.Error); ok { + if neterr.Temporary() { + retry = true + } + } + + // Retry if we get a 5xx series error. + if res != nil { + if res.StatusCode >= 500 && res.StatusCode < 600 { + retry = true + } + } + + return retry +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go new file mode 100644 index 000000000..2f6b39cf3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go @@ -0,0 +1,121 @@ +package aws_test + +import ( + "fmt" + "github.com/mitchellh/goamz/aws" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Retrieve the response from handler using aws.RetryingClient +func serveAndGet(handler http.HandlerFunc) (body string, err error) { + ts := httptest.NewServer(handler) + defer ts.Close() + resp, err := aws.RetryingClient.Get(ts.URL) + if err != nil { + return + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("Bad status code: %d", resp.StatusCode) + } + greeting, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return + } + return strings.TrimSpace(string(greeting)), nil +} + +func TestClient_expected(t *testing.T) { + body := "foo bar" + + resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, body) + }) + if err != nil { + t.Fatal(err) + } + if resp != body { + t.Fatal("Body not as expected.") + } +} + +func TestClient_delay(t *testing.T) { + body := "baz" + wait := 4 + resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { + if wait < 0 { + // If we dipped to zero delay and still failed. + t.Fatal("Never succeeded.") + } + wait -= 1 + time.Sleep(time.Second * time.Duration(wait)) + fmt.Fprintln(w, body) + }) + if err != nil { + t.Fatal(err) + } + if resp != body { + t.Fatal("Body not as expected.", resp) + } +} + +func TestClient_no4xxRetry(t *testing.T) { + tries := 0 + + // Fail once before succeeding. + _, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { + tries += 1 + http.Error(w, "error", 404) + }) + + if err == nil { + t.Fatal("should have error") + } + + if tries != 1 { + t.Fatalf("should only try once: %d", tries) + } +} + +func TestClient_retries(t *testing.T) { + body := "biz" + failed := false + // Fail once before succeeding. + resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { + if !failed { + http.Error(w, "error", 500) + failed = true + } else { + fmt.Fprintln(w, body) + } + }) + if failed != true { + t.Error("We didn't retry!") + } + if err != nil { + t.Fatal(err) + } + if resp != body { + t.Fatal("Body not as expected.") + } +} + +func TestClient_fails(t *testing.T) { + tries := 0 + // Fail 3 times and return the last error. + _, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { + tries += 1 + http.Error(w, "error", 500) + }) + if err == nil { + t.Fatal(err) + } + if tries != 3 { + t.Fatal("Didn't retry enough") + } +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go new file mode 100644 index 000000000..15bbd6762 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go @@ -0,0 +1,27 @@ +package s3 + +import ( + "github.com/mitchellh/goamz/aws" +) + +var originalStrategy = attempts + +func SetAttemptStrategy(s *aws.AttemptStrategy) { + if s == nil { + attempts = originalStrategy + } else { + attempts = *s + } +} + +func Sign(auth aws.Auth, method, path string, params, headers map[string][]string) { + sign(auth, method, path, params, headers) +} + +func SetListPartsMax(n int) { + listPartsMax = n +} + +func SetListMultiMax(n int) { + listMultiMax = n +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go new file mode 100644 index 000000000..6e318b31d --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go @@ -0,0 +1,409 @@ +package s3 + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "io" + "sort" + "strconv" +) + +// Multi represents an unfinished multipart upload. +// +// Multipart uploads allow sending big objects in smaller chunks. +// After all parts have been sent, the upload must be explicitly +// completed by calling Complete with the list of parts. +// +// See http://goo.gl/vJfTG for an overview of multipart uploads. +type Multi struct { + Bucket *Bucket + Key string + UploadId string +} + +// That's the default. Here just for testing. +var listMultiMax = 1000 + +type listMultiResp struct { + NextKeyMarker string + NextUploadIdMarker string + IsTruncated bool + Upload []Multi + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` +} + +// ListMulti returns the list of unfinished multipart uploads in b. +// +// The prefix parameter limits the response to keys that begin with the +// specified prefix. You can use prefixes to separate a bucket into different +// groupings of keys (to get the feeling of folders, for example). +// +// The delim parameter causes the response to group all of the keys that +// share a common prefix up to the next delimiter in a single entry within +// the CommonPrefixes field. You can use delimiters to separate a bucket +// into different groupings of keys, similar to how folders would work. +// +// See http://goo.gl/ePioY for details. +func (b *Bucket) ListMulti(prefix, delim string) (multis []*Multi, prefixes []string, err error) { + params := map[string][]string{ + "uploads": {""}, + "max-uploads": {strconv.FormatInt(int64(listMultiMax), 10)}, + "prefix": {prefix}, + "delimiter": {delim}, + } + for attempt := attempts.Start(); attempt.Next(); { + req := &request{ + method: "GET", + bucket: b.Name, + params: params, + } + var resp listMultiResp + err := b.S3.query(req, &resp) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, nil, err + } + for i := range resp.Upload { + multi := &resp.Upload[i] + multi.Bucket = b + multis = append(multis, multi) + } + prefixes = append(prefixes, resp.CommonPrefixes...) + if !resp.IsTruncated { + return multis, prefixes, nil + } + params["key-marker"] = []string{resp.NextKeyMarker} + params["upload-id-marker"] = []string{resp.NextUploadIdMarker} + attempt = attempts.Start() // Last request worked. + } + panic("unreachable") +} + +// Multi returns a multipart upload handler for the provided key +// inside b. If a multipart upload exists for key, it is returned, +// otherwise a new multipart upload is initiated with contType and perm. +func (b *Bucket) Multi(key, contType string, perm ACL) (*Multi, error) { + multis, _, err := b.ListMulti(key, "") + if err != nil && !hasCode(err, "NoSuchUpload") { + return nil, err + } + for _, m := range multis { + if m.Key == key { + return m, nil + } + } + return b.InitMulti(key, contType, perm) +} + +// InitMulti initializes a new multipart upload at the provided +// key inside b and returns a value for manipulating it. +// +// See http://goo.gl/XP8kL for details. +func (b *Bucket) InitMulti(key string, contType string, perm ACL) (*Multi, error) { + headers := map[string][]string{ + "Content-Type": {contType}, + "Content-Length": {"0"}, + "x-amz-acl": {string(perm)}, + } + params := map[string][]string{ + "uploads": {""}, + } + req := &request{ + method: "POST", + bucket: b.Name, + path: key, + headers: headers, + params: params, + } + var err error + var resp struct { + UploadId string `xml:"UploadId"` + } + for attempt := attempts.Start(); attempt.Next(); { + err = b.S3.query(req, &resp) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return &Multi{Bucket: b, Key: key, UploadId: resp.UploadId}, nil +} + +// PutPart sends part n of the multipart upload, reading all the content from r. +// Each part, except for the last one, must be at least 5MB in size. +// +// See http://goo.gl/pqZer for details. +func (m *Multi) PutPart(n int, r io.ReadSeeker) (Part, error) { + partSize, _, md5b64, err := seekerInfo(r) + if err != nil { + return Part{}, err + } + return m.putPart(n, r, partSize, md5b64) +} + +func (m *Multi) putPart(n int, r io.ReadSeeker, partSize int64, md5b64 string) (Part, error) { + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(partSize, 10)}, + "Content-MD5": {md5b64}, + } + params := map[string][]string{ + "uploadId": {m.UploadId}, + "partNumber": {strconv.FormatInt(int64(n), 10)}, + } + for attempt := attempts.Start(); attempt.Next(); { + _, err := r.Seek(0, 0) + if err != nil { + return Part{}, err + } + req := &request{ + method: "PUT", + bucket: m.Bucket.Name, + path: m.Key, + headers: headers, + params: params, + payload: r, + } + err = m.Bucket.S3.prepare(req) + if err != nil { + return Part{}, err + } + resp, err := m.Bucket.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return Part{}, err + } + etag := resp.Header.Get("ETag") + if etag == "" { + return Part{}, errors.New("part upload succeeded with no ETag") + } + return Part{n, etag, partSize}, nil + } + panic("unreachable") +} + +func seekerInfo(r io.ReadSeeker) (size int64, md5hex string, md5b64 string, err error) { + _, err = r.Seek(0, 0) + if err != nil { + return 0, "", "", err + } + digest := md5.New() + size, err = io.Copy(digest, r) + if err != nil { + return 0, "", "", err + } + sum := digest.Sum(nil) + md5hex = hex.EncodeToString(sum) + md5b64 = base64.StdEncoding.EncodeToString(sum) + return size, md5hex, md5b64, nil +} + +type Part struct { + N int `xml:"PartNumber"` + ETag string + Size int64 +} + +type partSlice []Part + +func (s partSlice) Len() int { return len(s) } +func (s partSlice) Less(i, j int) bool { return s[i].N < s[j].N } +func (s partSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type listPartsResp struct { + NextPartNumberMarker string + IsTruncated bool + Part []Part +} + +// That's the default. Here just for testing. +var listPartsMax = 1000 + +// ListParts returns the list of previously uploaded parts in m, +// ordered by part number. +// +// See http://goo.gl/ePioY for details. +func (m *Multi) ListParts() ([]Part, error) { + params := map[string][]string{ + "uploadId": {m.UploadId}, + "max-parts": {strconv.FormatInt(int64(listPartsMax), 10)}, + } + var parts partSlice + for attempt := attempts.Start(); attempt.Next(); { + req := &request{ + method: "GET", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + } + var resp listPartsResp + err := m.Bucket.S3.query(req, &resp) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + parts = append(parts, resp.Part...) + if !resp.IsTruncated { + sort.Sort(parts) + return parts, nil + } + params["part-number-marker"] = []string{resp.NextPartNumberMarker} + attempt = attempts.Start() // Last request worked. + } + panic("unreachable") +} + +type ReaderAtSeeker interface { + io.ReaderAt + io.ReadSeeker +} + +// PutAll sends all of r via a multipart upload with parts no larger +// than partSize bytes, which must be set to at least 5MB. +// Parts previously uploaded are either reused if their checksum +// and size match the new part, or otherwise overwritten with the +// new content. +// PutAll returns all the parts of m (reused or not). +func (m *Multi) PutAll(r ReaderAtSeeker, partSize int64) ([]Part, error) { + old, err := m.ListParts() + if err != nil && !hasCode(err, "NoSuchUpload") { + return nil, err + } + reuse := 0 // Index of next old part to consider reusing. + current := 1 // Part number of latest good part handled. + totalSize, err := r.Seek(0, 2) + if err != nil { + return nil, err + } + first := true // Must send at least one empty part if the file is empty. + var result []Part +NextSection: + for offset := int64(0); offset < totalSize || first; offset += partSize { + first = false + if offset+partSize > totalSize { + partSize = totalSize - offset + } + section := io.NewSectionReader(r, offset, partSize) + _, md5hex, md5b64, err := seekerInfo(section) + if err != nil { + return nil, err + } + for reuse < len(old) && old[reuse].N <= current { + // Looks like this part was already sent. + part := &old[reuse] + etag := `"` + md5hex + `"` + if part.N == current && part.Size == partSize && part.ETag == etag { + // Checksum matches. Reuse the old part. + result = append(result, *part) + current++ + continue NextSection + } + reuse++ + } + + // Part wasn't found or doesn't match. Send it. + part, err := m.putPart(current, section, partSize, md5b64) + if err != nil { + return nil, err + } + result = append(result, part) + current++ + } + return result, nil +} + +type completeUpload struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts completeParts `xml:"Part"` +} + +type completePart struct { + PartNumber int + ETag string +} + +type completeParts []completePart + +func (p completeParts) Len() int { return len(p) } +func (p completeParts) Less(i, j int) bool { return p[i].PartNumber < p[j].PartNumber } +func (p completeParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// Complete assembles the given previously uploaded parts into the +// final object. This operation may take several minutes. +// +// See http://goo.gl/2Z7Tw for details. +func (m *Multi) Complete(parts []Part) error { + params := map[string][]string{ + "uploadId": {m.UploadId}, + } + c := completeUpload{} + for _, p := range parts { + c.Parts = append(c.Parts, completePart{p.N, p.ETag}) + } + sort.Sort(c.Parts) + data, err := xml.Marshal(&c) + if err != nil { + return err + } + for attempt := attempts.Start(); attempt.Next(); { + req := &request{ + method: "POST", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + payload: bytes.NewReader(data), + } + err := m.Bucket.S3.query(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + return err + } + panic("unreachable") +} + +// Abort deletes an unifinished multipart upload and any previously +// uploaded parts for it. +// +// After a multipart upload is aborted, no additional parts can be +// uploaded using it. However, if any part uploads are currently in +// progress, those part uploads might or might not succeed. As a result, +// it might be necessary to abort a given multipart upload multiple +// times in order to completely free all storage consumed by all parts. +// +// NOTE: If the described scenario happens to you, please report back to +// the goamz authors with details. In the future such retrying should be +// handled internally, but it's not clear what happens precisely (Is an +// error returned? Is the issue completely undetectable?). +// +// See http://goo.gl/dnyJw for details. +func (m *Multi) Abort() error { + params := map[string][]string{ + "uploadId": {m.UploadId}, + } + for attempt := attempts.Start(); attempt.Next(); { + req := &request{ + method: "DELETE", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + } + err := m.Bucket.S3.query(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + return err + } + panic("unreachable") +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go new file mode 100644 index 000000000..fbb03e046 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go @@ -0,0 +1,370 @@ +package s3_test + +import ( + "encoding/xml" + "github.com/mitchellh/goamz/s3" + . "github.com/motain/gocheck" + "io" + "io/ioutil" + "strings" +) + +func (s *S) TestInitMulti(c *C) { + testServer.Response(200, nil, InitMultiResultDump) + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "POST") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Header["Content-Type"], DeepEquals, []string{"text/plain"}) + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) + c.Assert(req.Form["uploads"], DeepEquals, []string{""}) + + c.Assert(multi.UploadId, Matches, "JNbR_[A-Za-z0-9.]+QQ--") +} + +func (s *S) TestMultiNoPreviousUpload(c *C) { + // Don't retry the NoSuchUpload error. + s.DisableRetries() + + testServer.Response(404, nil, NoSuchUploadErrorDump) + testServer.Response(200, nil, InitMultiResultDump) + + b := s.s3.Bucket("sample") + + multi, err := b.Multi("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/") + c.Assert(req.Form["uploads"], DeepEquals, []string{""}) + c.Assert(req.Form["prefix"], DeepEquals, []string{"multi"}) + + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "POST") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["uploads"], DeepEquals, []string{""}) + + c.Assert(multi.UploadId, Matches, "JNbR_[A-Za-z0-9.]+QQ--") +} + +func (s *S) TestMultiReturnOld(c *C) { + testServer.Response(200, nil, ListMultiResultDump) + + b := s.s3.Bucket("sample") + + multi, err := b.Multi("multi1", "text/plain", s3.Private) + c.Assert(err, IsNil) + c.Assert(multi.Key, Equals, "multi1") + c.Assert(multi.UploadId, Equals, "iUVug89pPvSswrikD") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/") + c.Assert(req.Form["uploads"], DeepEquals, []string{""}) + c.Assert(req.Form["prefix"], DeepEquals, []string{"multi1"}) +} + +func (s *S) TestListParts(c *C) { + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(200, nil, ListPartsResultDump1) + testServer.Response(404, nil, NoSuchUploadErrorDump) // :-( + testServer.Response(200, nil, ListPartsResultDump2) + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + parts, err := multi.ListParts() + c.Assert(err, IsNil) + c.Assert(parts, HasLen, 3) + c.Assert(parts[0].N, Equals, 1) + c.Assert(parts[0].Size, Equals, int64(5)) + c.Assert(parts[0].ETag, Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`) + c.Assert(parts[1].N, Equals, 2) + c.Assert(parts[1].Size, Equals, int64(5)) + c.Assert(parts[1].ETag, Equals, `"d067a0fa9dc61a6e7195ca99696b5a89"`) + c.Assert(parts[2].N, Equals, 3) + c.Assert(parts[2].Size, Equals, int64(5)) + c.Assert(parts[2].ETag, Equals, `"49dcd91231f801159e893fb5c6674985"`) + testServer.WaitRequest() + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") + c.Assert(req.Form["max-parts"], DeepEquals, []string{"1000"}) + + testServer.WaitRequest() // The internal error. + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") + c.Assert(req.Form["max-parts"], DeepEquals, []string{"1000"}) + c.Assert(req.Form["part-number-marker"], DeepEquals, []string{"2"}) +} + +func (s *S) TestPutPart(c *C) { + headers := map[string]string{ + "ETag": `"26f90efd10d614f100252ff56d88dad8"`, + } + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(200, headers, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + part, err := multi.PutPart(1, strings.NewReader("")) + c.Assert(err, IsNil) + c.Assert(part.N, Equals, 1) + c.Assert(part.Size, Equals, int64(8)) + c.Assert(part.ETag, Equals, headers["ETag"]) + + testServer.WaitRequest() + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"1"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"8"}) + c.Assert(req.Header["Content-Md5"], DeepEquals, []string{"JvkO/RDWFPEAJS/1bYja2A=="}) +} + +func readAll(r io.Reader) string { + data, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + return string(data) +} + +func (s *S) TestPutAllNoPreviousUpload(c *C) { + // Don't retry the NoSuchUpload error. + s.DisableRetries() + + etag1 := map[string]string{"ETag": `"etag1"`} + etag2 := map[string]string{"ETag": `"etag2"`} + etag3 := map[string]string{"ETag": `"etag3"`} + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(404, nil, NoSuchUploadErrorDump) + testServer.Response(200, etag1, "") + testServer.Response(200, etag2, "") + testServer.Response(200, etag3, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5) + c.Assert(parts, HasLen, 3) + c.Assert(parts[0].ETag, Equals, `"etag1"`) + c.Assert(parts[1].ETag, Equals, `"etag2"`) + c.Assert(parts[2].ETag, Equals, `"etag3"`) + c.Assert(err, IsNil) + + // Init + testServer.WaitRequest() + + // List old parts. Won't find anything. + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/multi") + + // Send part 1. + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"1"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"5"}) + c.Assert(readAll(req.Body), Equals, "part1") + + // Send part 2. + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"2"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"5"}) + c.Assert(readAll(req.Body), Equals, "part2") + + // Send part 3 with shorter body. + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"3"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"4"}) + c.Assert(readAll(req.Body), Equals, "last") +} + +func (s *S) TestPutAllZeroSizeFile(c *C) { + // Don't retry the NoSuchUpload error. + s.DisableRetries() + + etag1 := map[string]string{"ETag": `"etag1"`} + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(404, nil, NoSuchUploadErrorDump) + testServer.Response(200, etag1, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + // Must send at least one part, so that completing it will work. + parts, err := multi.PutAll(strings.NewReader(""), 5) + c.Assert(parts, HasLen, 1) + c.Assert(parts[0].ETag, Equals, `"etag1"`) + c.Assert(err, IsNil) + + // Init + testServer.WaitRequest() + + // List old parts. Won't find anything. + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/multi") + + // Send empty part. + req = testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"1"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"0"}) + c.Assert(readAll(req.Body), Equals, "") +} + +func (s *S) TestPutAllResume(c *C) { + etag2 := map[string]string{"ETag": `"etag2"`} + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(200, nil, ListPartsResultDump1) + testServer.Response(200, nil, ListPartsResultDump2) + testServer.Response(200, etag2, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + // "part1" and "part3" match the checksums in ResultDump1. + // The middle one is a mismatch (it refers to "part2"). + parts, err := multi.PutAll(strings.NewReader("part1partXpart3"), 5) + c.Assert(parts, HasLen, 3) + c.Assert(parts[0].N, Equals, 1) + c.Assert(parts[0].Size, Equals, int64(5)) + c.Assert(parts[0].ETag, Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`) + c.Assert(parts[1].N, Equals, 2) + c.Assert(parts[1].Size, Equals, int64(5)) + c.Assert(parts[1].ETag, Equals, `"etag2"`) + c.Assert(parts[2].N, Equals, 3) + c.Assert(parts[2].Size, Equals, int64(5)) + c.Assert(parts[2].ETag, Equals, `"49dcd91231f801159e893fb5c6674985"`) + c.Assert(err, IsNil) + + // Init + testServer.WaitRequest() + + // List old parts, broken in two requests. + for i := 0; i < 2; i++ { + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/multi") + } + + // Send part 2, as it didn't match the checksum. + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form["partNumber"], DeepEquals, []string{"2"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"5"}) + c.Assert(readAll(req.Body), Equals, "partX") +} + +func (s *S) TestMultiComplete(c *C) { + testServer.Response(200, nil, InitMultiResultDump) + // Note the 200 response. Completing will hold the connection on some + // kind of long poll, and may return a late error even after a 200. + testServer.Response(200, nil, InternalErrorDump) + testServer.Response(200, nil, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) + c.Assert(err, IsNil) + + testServer.WaitRequest() + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "POST") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") + + var payload struct { + XMLName xml.Name + Part []struct { + PartNumber int + ETag string + } + } + + dec := xml.NewDecoder(req.Body) + err = dec.Decode(&payload) + c.Assert(err, IsNil) + + c.Assert(payload.XMLName.Local, Equals, "CompleteMultipartUpload") + c.Assert(len(payload.Part), Equals, 2) + c.Assert(payload.Part[0].PartNumber, Equals, 1) + c.Assert(payload.Part[0].ETag, Equals, `"ETag1"`) + c.Assert(payload.Part[1].PartNumber, Equals, 2) + c.Assert(payload.Part[1].ETag, Equals, `"ETag2"`) +} + +func (s *S) TestMultiAbort(c *C) { + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(200, nil, "") + + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + + err = multi.Abort() + c.Assert(err, IsNil) + + testServer.WaitRequest() + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "DELETE") + c.Assert(req.URL.Path, Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") +} + +func (s *S) TestListMulti(c *C) { + testServer.Response(200, nil, ListMultiResultDump) + + b := s.s3.Bucket("sample") + + multis, prefixes, err := b.ListMulti("", "/") + c.Assert(err, IsNil) + c.Assert(prefixes, DeepEquals, []string{"a/", "b/"}) + c.Assert(multis, HasLen, 2) + c.Assert(multis[0].Key, Equals, "multi1") + c.Assert(multis[0].UploadId, Equals, "iUVug89pPvSswrikD") + c.Assert(multis[1].Key, Equals, "multi2") + c.Assert(multis[1].UploadId, Equals, "DkirwsSvPp98guVUi") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/sample/") + c.Assert(req.Form["uploads"], DeepEquals, []string{""}) + c.Assert(req.Form["prefix"], DeepEquals, []string{""}) + c.Assert(req.Form["delimiter"], DeepEquals, []string{"/"}) + c.Assert(req.Form["max-uploads"], DeepEquals, []string{"1000"}) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go new file mode 100644 index 000000000..98bba563e --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go @@ -0,0 +1,241 @@ +package s3_test + +var GetObjectErrorDump = ` + +NoSuchBucketThe specified bucket does not exist +non-existent-bucket3F1B667FAD71C3D8 +L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D +` + +var GetListResultDump1 = ` + + + quotes + N + false + + Nelson + 2006-01-01T12:00:00.000Z + "828ef3fdfa96f00ad9f27c383fc9ac7f" + 5 + STANDARD + + bcaf161ca5fb16fd081034f + webfile + + + + Neo + 2006-01-01T12:00:00.000Z + "828ef3fdfa96f00ad9f27c383fc9ac7f" + 4 + STANDARD + + bcaf1ffd86a5fb16fd081034f + webfile + + + +` + +var GetListResultDump2 = ` + + example-bucket + photos/2006/ + some-marker + 1000 + / + false + + + photos/2006/feb/ + + + photos/2006/jan/ + + +` + +var InitMultiResultDump = ` + + + sample + multi + JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- + +` + +var ListPartsResultDump1 = ` + + + sample + multi + JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- + + bb5c0f63b0b25f2d099c + joe + + + bb5c0f63b0b25f2d099c + joe + + STANDARD + 0 + 2 + 2 + true + + 1 + 2013-01-30T13:45:51.000Z + "ffc88b4ca90a355f8ddba6b2c3b2af5c" + 5 + + + 2 + 2013-01-30T13:45:52.000Z + "d067a0fa9dc61a6e7195ca99696b5a89" + 5 + + +` + +var ListPartsResultDump2 = ` + + + sample + multi + JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- + + bb5c0f63b0b25f2d099c + joe + + + bb5c0f63b0b25f2d099c + joe + + STANDARD + 2 + 3 + 2 + false + + 3 + 2013-01-30T13:46:50.000Z + "49dcd91231f801159e893fb5c6674985" + 5 + + +` + +var ListMultiResultDump = ` + + + goamz-test-bucket-us-east-1-akiajk3wyewhctyqbf7a + + + multi1 + iUVug89pPvSswrikD72p8uO62EzhNtpDxRmwC5WSiWDdK9SfzmDqe3xpP1kMWimyimSnz4uzFc3waVM5ufrKYQ-- + / + 1000 + false + + multi1 + iUVug89pPvSswrikD + + bb5c0f63b0b25f2d0 + gustavoniemeyer + + + bb5c0f63b0b25f2d0 + gustavoniemeyer + + STANDARD + 2013-01-30T18:15:47.000Z + + + multi2 + DkirwsSvPp98guVUi + + bb5c0f63b0b25f2d0 + joe + + + bb5c0f63b0b25f2d0 + joe + + STANDARD + 2013-01-30T18:15:47.000Z + + + a/ + + + b/ + + +` + +var NoSuchUploadErrorDump = ` + + + NoSuchUpload + Not relevant + sample + 3F1B667FAD71C3D8 + kjhwqk + +` + +var InternalErrorDump = ` + + + InternalError + Not relevant + sample + 3F1B667FAD71C3D8 + kjhwqk + +` + +var GetKeyHeaderDump = map[string]string{ + "x-amz-id-2": "ef8yU9AS1ed4OpIszj7UDNEHGran", + "x-amz-request-id": "318BC8BC143432E5", + "x-amz-version-id": "3HL4kqtJlcpXroDTDmjVBH40Nrjfkd", + "Date": "Wed, 28 Oct 2009 22:32:00 GMT", + "Last-Modified": "Sun, 1 Jan 2006 12:00:00 GMT", + "ETag": "fba9dede5f27731c9771645a39863328", + "Content-Length": "434234", + "Content-Type": "text/plain", +} + +var GetListBucketsDump = ` + + + + bb5c0f63b0b25f2d0 + joe + + + + bucket1 + 2012-01-01T02:03:04.000Z + + + bucket2 + 2014-01-11T02:03:04.000Z + + + +` + +var MultiDelDump = ` + + + + a.go + + + b.go + + +` diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go new file mode 100644 index 000000000..dbeec6efa --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go @@ -0,0 +1,893 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// +// Written by Gustavo Niemeyer +// + +package s3 + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/xml" + "fmt" + "github.com/mitchellh/goamz/aws" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "time" +) + +const debug = false + +// The S3 type encapsulates operations with an S3 region. +type S3 struct { + aws.Auth + aws.Region + HTTPClient func() *http.Client + + private byte // Reserve the right of using private data. +} + +// The Bucket type encapsulates operations with an S3 bucket. +type Bucket struct { + *S3 + Name string +} + +// The Owner type represents the owner of the object in an S3 bucket. +type Owner struct { + ID string + DisplayName string +} + +var attempts = aws.AttemptStrategy{ + Min: 5, + Total: 5 * time.Second, + Delay: 200 * time.Millisecond, +} + +// New creates a new S3. +func New(auth aws.Auth, region aws.Region) *S3 { + return &S3{ + Auth: auth, + Region: region, + HTTPClient: func() *http.Client { + return http.DefaultClient + }, + private: 0} +} + +// Bucket returns a Bucket with the given name. +func (s3 *S3) Bucket(name string) *Bucket { + if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { + name = strings.ToLower(name) + } + return &Bucket{s3, name} +} + +var createBucketConfiguration = ` + %s +` + +// locationConstraint returns an io.Reader specifying a LocationConstraint if +// required for the region. +// +// See http://goo.gl/bh9Kq for details. +func (s3 *S3) locationConstraint() io.Reader { + constraint := "" + if s3.Region.S3LocationConstraint { + constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) + } + return strings.NewReader(constraint) +} + +type ACL string + +const ( + Private = ACL("private") + PublicRead = ACL("public-read") + PublicReadWrite = ACL("public-read-write") + AuthenticatedRead = ACL("authenticated-read") + BucketOwnerRead = ACL("bucket-owner-read") + BucketOwnerFull = ACL("bucket-owner-full-control") +) + +// The ListBucketsResp type holds the results of a List buckets operation. +type ListBucketsResp struct { + Buckets []Bucket `xml:">Bucket"` +} + +// ListBuckets lists all buckets +// +// See: http://goo.gl/NqlyMN +func (s3 *S3) ListBuckets() (result *ListBucketsResp, err error) { + req := &request{ + path: "/", + } + result = &ListBucketsResp{} + for attempt := attempts.Start(); attempt.Next(); { + err = s3.query(req, result) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + // set S3 instance on buckets + for i := range result.Buckets { + result.Buckets[i].S3 = s3 + } + return result, nil +} + +// PutBucket creates a new bucket. +// +// See http://goo.gl/ndjnR for details. +func (b *Bucket) PutBucket(perm ACL) error { + headers := map[string][]string{ + "x-amz-acl": {string(perm)}, + } + req := &request{ + method: "PUT", + bucket: b.Name, + path: "/", + headers: headers, + payload: b.locationConstraint(), + } + return b.S3.query(req, nil) +} + +// DelBucket removes an existing S3 bucket. All objects in the bucket must +// be removed before the bucket itself can be removed. +// +// See http://goo.gl/GoBrY for details. +func (b *Bucket) DelBucket() (err error) { + req := &request{ + method: "DELETE", + bucket: b.Name, + path: "/", + } + for attempt := attempts.Start(); attempt.Next(); { + err = b.S3.query(req, nil) + if !shouldRetry(err) { + break + } + } + return err +} + +// Get retrieves an object from an S3 bucket. +// +// See http://goo.gl/isCO7 for details. +func (b *Bucket) Get(path string) (data []byte, err error) { + body, err := b.GetReader(path) + if err != nil { + return nil, err + } + data, err = ioutil.ReadAll(body) + body.Close() + return data, err +} + +// GetReader retrieves an object from an S3 bucket. +// It is the caller's responsibility to call Close on rc when +// finished reading. +func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) { + resp, err := b.GetResponse(path) + if resp != nil { + return resp.Body, err + } + return nil, err +} + +// GetResponse retrieves an object from an S3 bucket returning the http response +// It is the caller's responsibility to call Close on rc when +// finished reading. +func (b *Bucket) GetResponse(path string) (*http.Response, error) { + return b.getResponseParams(path, nil) +} + +// GetTorrent retrieves an Torrent object from an S3 bucket an io.ReadCloser. +// It is the caller's responsibility to call Close on rc when finished reading. +func (b *Bucket) GetTorrentReader(path string) (io.ReadCloser, error) { + resp, err := b.getResponseParams(path, url.Values{"torrent": {""}}) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// GetTorrent retrieves an Torrent object from an S3, returning +// the torrent as a []byte. +func (b *Bucket) GetTorrent(path string) ([]byte, error) { + body, err := b.GetTorrentReader(path) + if err != nil { + return nil, err + } + defer body.Close() + + return ioutil.ReadAll(body) +} + +func (b *Bucket) getResponseParams(path string, params url.Values) (*http.Response, error) { + req := &request{ + bucket: b.Name, + path: path, + params: params, + } + err := b.S3.prepare(req) + if err != nil { + return nil, err + } + for attempt := attempts.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + return resp, nil + } + panic("unreachable") +} + +func (b *Bucket) Head(path string) (*http.Response, error) { + req := &request{ + method: "HEAD", + bucket: b.Name, + path: path, + } + err := b.S3.prepare(req) + if err != nil { + return nil, err + } + for attempt := attempts.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + return resp, nil + } + panic("unreachable") +} + +// Put inserts an object into the S3 bucket. +// +// See http://goo.gl/FEBPD for details. +func (b *Bucket) Put(path string, data []byte, contType string, perm ACL) error { + body := bytes.NewBuffer(data) + return b.PutReader(path, body, int64(len(data)), contType, perm) +} + +/* +PutHeader - like Put, inserts an object into the S3 bucket. +Instead of Content-Type string, pass in custom headers to override defaults. +*/ +func (b *Bucket) PutHeader(path string, data []byte, customHeaders map[string][]string, perm ACL) error { + body := bytes.NewBuffer(data) + return b.PutReaderHeader(path, body, int64(len(data)), customHeaders, perm) +} + +// PutReader inserts an object into the S3 bucket by consuming data +// from r until EOF. +func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL) error { + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(length, 10)}, + "Content-Type": {contType}, + "x-amz-acl": {string(perm)}, + } + req := &request{ + method: "PUT", + bucket: b.Name, + path: path, + headers: headers, + payload: r, + } + return b.S3.query(req, nil) +} + +/* +PutReaderHeader - like PutReader, inserts an object into S3 from a reader. +Instead of Content-Type string, pass in custom headers to override defaults. +*/ +func (b *Bucket) PutReaderHeader(path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error { + // Default headers + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(length, 10)}, + "Content-Type": {"application/text"}, + "x-amz-acl": {string(perm)}, + } + + // Override with custom headers + for key, value := range customHeaders { + headers[key] = value + } + + req := &request{ + method: "PUT", + bucket: b.Name, + path: path, + headers: headers, + payload: r, + } + return b.S3.query(req, nil) +} + +/* +Copy - copy objects inside bucket +*/ +func (b *Bucket) Copy(oldPath, newPath string, perm ACL) error { + if !strings.HasPrefix(oldPath, "/") { + oldPath = "/" + oldPath + } + + req := &request{ + method: "PUT", + bucket: b.Name, + path: newPath, + headers: map[string][]string{ + "x-amz-copy-source": {amazonEscape("/" + b.Name + oldPath)}, + "x-amz-acl": {string(perm)}, + }, + } + + err := b.S3.prepare(req) + if err != nil { + return err + } + + for attempt := attempts.Start(); attempt.Next(); { + _, err = b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return err + } + return nil + } + panic("unreachable") +} + +// Del removes an object from the S3 bucket. +// +// See http://goo.gl/APeTt for details. +func (b *Bucket) Del(path string) error { + req := &request{ + method: "DELETE", + bucket: b.Name, + path: path, + } + return b.S3.query(req, nil) +} + +type Object struct { + Key string +} + +type MultiObjectDeleteBody struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool + Object []Object +} + +func base64md5(data []byte) string { + h := md5.New() + h.Write(data) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +// MultiDel removes multiple objects from the S3 bucket efficiently. +// A maximum of 1000 keys at once may be specified. +// +// See http://goo.gl/WvA5sj for details. +func (b *Bucket) MultiDel(paths []string) error { + // create XML payload + v := MultiObjectDeleteBody{} + v.Object = make([]Object, len(paths)) + for i, path := range paths { + v.Object[i] = Object{path} + } + data, _ := xml.Marshal(v) + + // Content-MD5 is required + md5hash := base64md5(data) + req := &request{ + method: "POST", + bucket: b.Name, + path: "/", + params: url.Values{"delete": {""}}, + headers: http.Header{"Content-MD5": {md5hash}}, + payload: bytes.NewReader(data), + } + + return b.S3.query(req, nil) +} + +// The ListResp type holds the results of a List bucket operation. +type ListResp struct { + Name string + Prefix string + Delimiter string + Marker string + NextMarker string + MaxKeys int + // IsTruncated is true if the results have been truncated because + // there are more keys and prefixes than can fit in MaxKeys. + // N.B. this is the opposite sense to that documented (incorrectly) in + // http://goo.gl/YjQTc + IsTruncated bool + Contents []Key + CommonPrefixes []string `xml:">Prefix"` +} + +// The Key type represents an item stored in an S3 bucket. +type Key struct { + Key string + LastModified string + Size int64 + // ETag gives the hex-encoded MD5 sum of the contents, + // surrounded with double-quotes. + ETag string + StorageClass string + Owner Owner +} + +// List returns information about objects in an S3 bucket. +// +// The prefix parameter limits the response to keys that begin with the +// specified prefix. +// +// The delim parameter causes the response to group all of the keys that +// share a common prefix up to the next delimiter in a single entry within +// the CommonPrefixes field. You can use delimiters to separate a bucket +// into different groupings of keys, similar to how folders would work. +// +// The marker parameter specifies the key to start with when listing objects +// in a bucket. Amazon S3 lists objects in alphabetical order and +// will return keys alphabetically greater than the marker. +// +// The max parameter specifies how many keys + common prefixes to return in +// the response. The default is 1000. +// +// For example, given these keys in a bucket: +// +// index.html +// index2.html +// photos/2006/January/sample.jpg +// photos/2006/February/sample2.jpg +// photos/2006/February/sample3.jpg +// photos/2006/February/sample4.jpg +// +// Listing this bucket with delimiter set to "/" would yield the +// following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Contents: []Key{ +// {Key: "index.html", "index2.html"}, +// }, +// CommonPrefixes: []string{ +// "photos/", +// }, +// } +// +// Listing the same bucket with delimiter set to "/" and prefix set to +// "photos/2006/" would yield the following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Prefix: "photos/2006/", +// CommonPrefixes: []string{ +// "photos/2006/February/", +// "photos/2006/January/", +// }, +// } +// +// See http://goo.gl/YjQTc for details. +func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) { + params := map[string][]string{ + "prefix": {prefix}, + "delimiter": {delim}, + "marker": {marker}, + } + if max != 0 { + params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} + } + req := &request{ + bucket: b.Name, + params: params, + } + result = &ListResp{} + for attempt := attempts.Start(); attempt.Next(); { + err = b.S3.query(req, result) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return result, nil +} + +// Returns a mapping of all key names in this bucket to Key objects +func (b *Bucket) GetBucketContents() (*map[string]Key, error) { + bucket_contents := map[string]Key{} + prefix := "" + path_separator := "" + marker := "" + for { + contents, err := b.List(prefix, path_separator, marker, 1000) + if err != nil { + return &bucket_contents, err + } + last_key := "" + for _, key := range contents.Contents { + bucket_contents[key.Key] = key + last_key = key.Key + } + if contents.IsTruncated { + marker = contents.NextMarker + if marker == "" { + // From the s3 docs: If response does not include the + // NextMarker and it is truncated, you can use the value of the + // last Key in the response as the marker in the subsequent + // request to get the next set of object keys. + marker = last_key + } + } else { + break + } + } + + return &bucket_contents, nil +} + +// Get metadata from the key without returning the key content +func (b *Bucket) GetKey(path string) (*Key, error) { + req := &request{ + bucket: b.Name, + path: path, + method: "HEAD", + } + err := b.S3.prepare(req) + if err != nil { + return nil, err + } + key := &Key{} + for attempt := attempts.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + key.Key = path + key.LastModified = resp.Header.Get("Last-Modified") + key.ETag = resp.Header.Get("ETag") + contentLength := resp.Header.Get("Content-Length") + size, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return key, fmt.Errorf("bad s3 content-length %v: %v", + contentLength, err) + } + key.Size = size + return key, nil + } + panic("unreachable") +} + +// URL returns a non-signed URL that allows retriving the +// object at path. It only works if the object is publicly +// readable (see SignedURL). +func (b *Bucket) URL(path string) string { + req := &request{ + bucket: b.Name, + path: path, + } + err := b.S3.prepare(req) + if err != nil { + panic(err) + } + u, err := req.url(true) + if err != nil { + panic(err) + } + u.RawQuery = "" + return u.String() +} + +// SignedURL returns a signed URL that allows anyone holding the URL +// to retrieve the object at path. The signature is valid until expires. +func (b *Bucket) SignedURL(path string, expires time.Time) string { + req := &request{ + bucket: b.Name, + path: path, + params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}}, + } + err := b.S3.prepare(req) + if err != nil { + panic(err) + } + u, err := req.url(true) + if err != nil { + panic(err) + } + return u.String() +} + +type request struct { + method string + bucket string + path string + signpath string + params url.Values + headers http.Header + baseurl string + payload io.Reader + prepared bool +} + +// amazonShouldEscape returns true if byte should be escaped +func amazonShouldEscape(c byte) bool { + return !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '~' || c == '.' || c == '/' || c == ':') +} + +// amazonEscape does uri escaping exactly as Amazon does +func amazonEscape(s string) string { + hexCount := 0 + + for i := 0; i < len(s); i++ { + if amazonShouldEscape(s[i]) { + hexCount++ + } + } + + if hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + if c := s[i]; amazonShouldEscape(c) { + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + } else { + t[j] = s[i] + j++ + } + } + return string(t) +} + +// url returns url to resource, either full (with host/scheme) or +// partial for HTTP request +func (req *request) url(full bool) (*url.URL, error) { + u, err := url.Parse(req.baseurl) + if err != nil { + return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) + } + + u.Opaque = amazonEscape(req.path) + if full { + u.Opaque = "//" + u.Host + u.Opaque + } + u.RawQuery = req.params.Encode() + + return u, nil +} + +// query prepares and runs the req request. +// If resp is not nil, the XML data contained in the response +// body will be unmarshalled on it. +func (s3 *S3) query(req *request, resp interface{}) error { + err := s3.prepare(req) + if err == nil { + var httpResponse *http.Response + httpResponse, err = s3.run(req, resp) + if resp == nil && httpResponse != nil { + httpResponse.Body.Close() + } + } + return err +} + +// prepare sets up req to be delivered to S3. +func (s3 *S3) prepare(req *request) error { + if !req.prepared { + req.prepared = true + if req.method == "" { + req.method = "GET" + } + // Copy so they can be mutated without affecting on retries. + params := make(url.Values) + headers := make(http.Header) + for k, v := range req.params { + params[k] = v + } + for k, v := range req.headers { + headers[k] = v + } + req.params = params + req.headers = headers + if !strings.HasPrefix(req.path, "/") { + req.path = "/" + req.path + } + req.signpath = req.path + + if req.bucket != "" { + req.baseurl = s3.Region.S3BucketEndpoint + if req.baseurl == "" { + // Use the path method to address the bucket. + req.baseurl = s3.Region.S3Endpoint + req.path = "/" + req.bucket + req.path + } else { + // Just in case, prevent injection. + if strings.IndexAny(req.bucket, "/:@") >= 0 { + return fmt.Errorf("bad S3 bucket: %q", req.bucket) + } + req.baseurl = strings.Replace(req.baseurl, "${bucket}", req.bucket, -1) + } + req.signpath = "/" + req.bucket + req.signpath + } else { + req.baseurl = s3.Region.S3Endpoint + } + } + + // Always sign again as it's not clear how far the + // server has handled a previous attempt. + u, err := url.Parse(req.baseurl) + if err != nil { + return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) + } + req.headers["Host"] = []string{u.Host} + req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} + sign(s3.Auth, req.method, amazonEscape(req.signpath), req.params, req.headers) + return nil +} + +// run sends req and returns the http response from the server. +// If resp is not nil, the XML data contained in the response +// body will be unmarshalled on it. +func (s3 *S3) run(req *request, resp interface{}) (*http.Response, error) { + if debug { + log.Printf("Running S3 request: %#v", req) + } + + u, err := req.url(false) + if err != nil { + return nil, err + } + + hreq := http.Request{ + URL: u, + Method: req.method, + ProtoMajor: 1, + ProtoMinor: 1, + Close: true, + Header: req.headers, + } + + if v, ok := req.headers["Content-Length"]; ok { + hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64) + delete(req.headers, "Content-Length") + } + if req.payload != nil { + hreq.Body = ioutil.NopCloser(req.payload) + } + + hresp, err := s3.HTTPClient().Do(&hreq) + if err != nil { + return nil, err + } + if debug { + dump, _ := httputil.DumpResponse(hresp, true) + log.Printf("} -> %s\n", dump) + } + if hresp.StatusCode != 200 && hresp.StatusCode != 204 { + defer hresp.Body.Close() + return nil, buildError(hresp) + } + if resp != nil { + err = xml.NewDecoder(hresp.Body).Decode(resp) + hresp.Body.Close() + } + return hresp, err +} + +// Error represents an error in an operation with S3. +type Error struct { + StatusCode int // HTTP status code (200, 403, ...) + Code string // EC2 error code ("UnsupportedOperation", ...) + Message string // The human-oriented error message + BucketName string + RequestId string + HostId string +} + +func (e *Error) Error() string { + return e.Message +} + +func buildError(r *http.Response) error { + if debug { + log.Printf("got error (status code %v)", r.StatusCode) + data, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("\tread error: %v", err) + } else { + log.Printf("\tdata:\n%s\n\n", data) + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + } + + err := Error{} + // TODO return error if Unmarshal fails? + xml.NewDecoder(r.Body).Decode(&err) + r.Body.Close() + err.StatusCode = r.StatusCode + if err.Message == "" { + err.Message = r.Status + } + if debug { + log.Printf("err: %#v\n", err) + } + return &err +} + +func shouldRetry(err error) bool { + if err == nil { + return false + } + switch err { + case io.ErrUnexpectedEOF, io.EOF: + return true + } + switch e := err.(type) { + case *net.DNSError: + return true + case *net.OpError: + switch e.Op { + case "read", "write": + return true + } + case *Error: + switch e.Code { + case "InternalError", "NoSuchUpload", "NoSuchBucket": + return true + } + } + return false +} + +func hasCode(err error, code string) bool { + s3err, ok := err.(*Error) + return ok && s3err.Code == code +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go new file mode 100644 index 000000000..376c94700 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go @@ -0,0 +1,435 @@ +package s3_test + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "time" + + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/s3" + "github.com/mitchellh/goamz/testutil" + . "github.com/motain/gocheck" +) + +func Test(t *testing.T) { + TestingT(t) +} + +type S struct { + s3 *s3.S3 +} + +var _ = Suite(&S{}) + +var testServer = testutil.NewHTTPServer() + +func (s *S) SetUpSuite(c *C) { + testServer.Start() + auth := aws.Auth{"abc", "123", ""} + s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL}) +} + +func (s *S) TearDownSuite(c *C) { + s3.SetAttemptStrategy(nil) +} + +func (s *S) SetUpTest(c *C) { + attempts := aws.AttemptStrategy{ + Total: 300 * time.Millisecond, + Delay: 100 * time.Millisecond, + } + s3.SetAttemptStrategy(&attempts) +} + +func (s *S) TearDownTest(c *C) { + testServer.Flush() +} + +func (s *S) DisableRetries() { + s3.SetAttemptStrategy(&aws.AttemptStrategy{}) +} + +// PutBucket docs: http://goo.gl/kBTCu + +func (s *S) TestPutBucket(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/") + c.Assert(req.Header["Date"], Not(Equals), "") +} + +// DeleteBucket docs: http://goo.gl/GoBrY + +func (s *S) TestDelBucket(c *C) { + testServer.Response(204, nil, "") + + b := s.s3.Bucket("bucket") + err := b.DelBucket() + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "DELETE") + c.Assert(req.URL.Path, Equals, "/bucket/") + c.Assert(req.Header["Date"], Not(Equals), "") +} + +// ListBuckets: http://goo.gl/NqlyMN + +func (s *S) TestListBuckets(c *C) { + testServer.Response(200, nil, GetListBucketsDump) + + buckets, err := s.s3.ListBuckets() + c.Assert(err, IsNil) + c.Assert(len(buckets.Buckets), Equals, 2) + c.Assert(buckets.Buckets[0].Name, Equals, "bucket1") + c.Assert(buckets.Buckets[1].Name, Equals, "bucket2") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/") +} + +// GetObject docs: http://goo.gl/isCO7 + +func (s *S) TestGet(c *C) { + testServer.Response(200, nil, "content") + + b := s.s3.Bucket("bucket") + data, err := b.Get("name") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") + + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "content") +} + +func (s *S) TestHead(c *C) { + testServer.Response(200, nil, "") + b := s.s3.Bucket("bucket") + resp, err := b.Head("name") + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "HEAD") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") + + c.Assert(err, IsNil) + body, err := ioutil.ReadAll(resp.Body) + c.Assert(err, IsNil) + c.Assert(len(body), Equals, 0) +} + +func (s *S) TestURL(c *C) { + testServer.Response(200, nil, "content") + + b := s.s3.Bucket("bucket") + url := b.URL("name") + r, err := http.Get(url) + c.Assert(err, IsNil) + data, err := ioutil.ReadAll(r.Body) + r.Body.Close() + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "content") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/bucket/name") +} + +func (s *S) TestGetReader(c *C) { + testServer.Response(200, nil, "content") + + b := s.s3.Bucket("bucket") + rc, err := b.GetReader("name") + c.Assert(err, IsNil) + data, err := ioutil.ReadAll(rc) + rc.Close() + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "content") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") +} + +func (s *S) TestGetNotFound(c *C) { + for i := 0; i < 10; i++ { + testServer.Response(404, nil, GetObjectErrorDump) + } + + b := s.s3.Bucket("non-existent-bucket") + data, err := b.Get("non-existent") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/non-existent-bucket/non-existent") + c.Assert(req.Header["Date"], Not(Equals), "") + + s3err, _ := err.(*s3.Error) + c.Assert(s3err, NotNil) + c.Assert(s3err.StatusCode, Equals, 404) + c.Assert(s3err.BucketName, Equals, "non-existent-bucket") + c.Assert(s3err.RequestId, Equals, "3F1B667FAD71C3D8") + c.Assert(s3err.HostId, Equals, "L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D") + c.Assert(s3err.Code, Equals, "NoSuchBucket") + c.Assert(s3err.Message, Equals, "The specified bucket does not exist") + c.Assert(s3err.Error(), Equals, "The specified bucket does not exist") + c.Assert(data, IsNil) +} + +// PutObject docs: http://goo.gl/FEBPD + +func (s *S) TestPutObject(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.Put("name", []byte("content"), "content-type", s3.Private) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) + c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) + //c.Assert(req.Header["Content-MD5"], DeepEquals, "...") + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +func (s *S) TestPutObjectHeader(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.PutHeader( + "name", + []byte("content"), + map[string][]string{"Content-Type": {"content-type"}}, + s3.Private, + ) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) + c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) + //c.Assert(req.Header["Content-MD5"], DeepEquals, "...") + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +func (s *S) TestPutReader(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + buf := bytes.NewBufferString("content") + err := b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) + c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) + //c.Assert(req.Header["Content-MD5"], Equals, "...") + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +func (s *S) TestPutReaderHeader(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + buf := bytes.NewBufferString("content") + err := b.PutReaderHeader( + "name", + buf, + int64(buf.Len()), + map[string][]string{"Content-Type": {"content-type"}}, + s3.Private, + ) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) + c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) + c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) + //c.Assert(req.Header["Content-MD5"], Equals, "...") + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +func (s *S) TestCopy(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.Copy( + "old/file", + "new/file", + s3.Private, + ) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.URL.Path, Equals, "/bucket/new/file") + c.Assert(req.Header["X-Amz-Copy-Source"], DeepEquals, []string{"/bucket/old/file"}) + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +func (s *S) TestPlusInURL(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.Copy( + "dir/old+f?le", + "dir/new+f?le", + s3.Private, + ) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "PUT") + c.Assert(req.RequestURI, Equals, "/bucket/dir/new%2Bf%3Fle") + c.Assert(req.Header["X-Amz-Copy-Source"], DeepEquals, []string{"/bucket/dir/old%2Bf%3Fle"}) + c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) +} + +// DelObject docs: http://goo.gl/APeTt + +func (s *S) TestDelObject(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.Del("name") + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "DELETE") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") +} + +// Delete Multiple Objects docs: http://goo.gl/WvA5sj + +func (s *S) TestMultiDelObject(c *C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.MultiDel([]string{"a", "b"}) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "POST") + c.Assert(req.URL.Path, Equals, "/bucket/") + c.Assert(req.RequestURI, Equals, "/bucket/?delete=") + c.Assert(req.Header["Content-Md5"], DeepEquals, []string{"nos/vZNvjGs17xIyjEFlwQ=="}) + data, err := ioutil.ReadAll(req.Body) + req.Body.Close() + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "falseab") +} + +// Bucket List Objects docs: http://goo.gl/YjQTc + +func (s *S) TestList(c *C) { + testServer.Response(200, nil, GetListResultDump1) + + b := s.s3.Bucket("quotes") + + data, err := b.List("N", "", "", 0) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/quotes/") + c.Assert(req.Header["Date"], Not(Equals), "") + c.Assert(req.Form["prefix"], DeepEquals, []string{"N"}) + c.Assert(req.Form["delimiter"], DeepEquals, []string{""}) + c.Assert(req.Form["marker"], DeepEquals, []string{""}) + c.Assert(req.Form["max-keys"], DeepEquals, []string(nil)) + + c.Assert(data.Name, Equals, "quotes") + c.Assert(data.Prefix, Equals, "N") + c.Assert(data.IsTruncated, Equals, false) + c.Assert(len(data.Contents), Equals, 2) + + c.Assert(data.Contents[0].Key, Equals, "Nelson") + c.Assert(data.Contents[0].LastModified, Equals, "2006-01-01T12:00:00.000Z") + c.Assert(data.Contents[0].ETag, Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`) + c.Assert(data.Contents[0].Size, Equals, int64(5)) + c.Assert(data.Contents[0].StorageClass, Equals, "STANDARD") + c.Assert(data.Contents[0].Owner.ID, Equals, "bcaf161ca5fb16fd081034f") + c.Assert(data.Contents[0].Owner.DisplayName, Equals, "webfile") + + c.Assert(data.Contents[1].Key, Equals, "Neo") + c.Assert(data.Contents[1].LastModified, Equals, "2006-01-01T12:00:00.000Z") + c.Assert(data.Contents[1].ETag, Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`) + c.Assert(data.Contents[1].Size, Equals, int64(4)) + c.Assert(data.Contents[1].StorageClass, Equals, "STANDARD") + c.Assert(data.Contents[1].Owner.ID, Equals, "bcaf1ffd86a5fb16fd081034f") + c.Assert(data.Contents[1].Owner.DisplayName, Equals, "webfile") +} + +func (s *S) TestListWithDelimiter(c *C) { + testServer.Response(200, nil, GetListResultDump2) + + b := s.s3.Bucket("quotes") + + data, err := b.List("photos/2006/", "/", "some-marker", 1000) + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/quotes/") + c.Assert(req.Header["Date"], Not(Equals), "") + c.Assert(req.Form["prefix"], DeepEquals, []string{"photos/2006/"}) + c.Assert(req.Form["delimiter"], DeepEquals, []string{"/"}) + c.Assert(req.Form["marker"], DeepEquals, []string{"some-marker"}) + c.Assert(req.Form["max-keys"], DeepEquals, []string{"1000"}) + + c.Assert(data.Name, Equals, "example-bucket") + c.Assert(data.Prefix, Equals, "photos/2006/") + c.Assert(data.Delimiter, Equals, "/") + c.Assert(data.Marker, Equals, "some-marker") + c.Assert(data.IsTruncated, Equals, false) + c.Assert(len(data.Contents), Equals, 0) + c.Assert(data.CommonPrefixes, DeepEquals, []string{"photos/2006/feb/", "photos/2006/jan/"}) +} + +func (s *S) TestGetKey(c *C) { + testServer.Response(200, GetKeyHeaderDump, "") + + b := s.s3.Bucket("bucket") + key, err := b.GetKey("name") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "HEAD") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") + + c.Assert(err, IsNil) + c.Assert(key.Key, Equals, "name") + c.Assert(key.LastModified, Equals, GetKeyHeaderDump["Last-Modified"]) + c.Assert(key.Size, Equals, int64(434234)) + c.Assert(key.ETag, Equals, GetKeyHeaderDump["ETag"]) +} + +func (s *S) TestUnescapedColon(c *C) { + b := s.s3.Bucket("bucket") + u := b.URL("foo:bar") + c.Assert(u, Equals, "http://localhost:4444/bucket/foo:bar") +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go new file mode 100644 index 000000000..9b3145d8a --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go @@ -0,0 +1,616 @@ +package s3_test + +import ( + "bytes" + "crypto/md5" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/s3" + "github.com/mitchellh/goamz/testutil" + . "github.com/motain/gocheck" + "net" + "sort" + "time" +) + +// AmazonServer represents an Amazon S3 server. +type AmazonServer struct { + auth aws.Auth +} + +func (s *AmazonServer) SetUp(c *C) { + auth, err := aws.EnvAuth() + if err != nil { + c.Fatal(err.Error()) + } + s.auth = auth +} + +var _ = Suite(&AmazonClientSuite{Region: aws.USEast}) +var _ = Suite(&AmazonClientSuite{Region: aws.EUWest}) +var _ = Suite(&AmazonClientSuite{Region: aws.EUCentral}) +var _ = Suite(&AmazonDomainClientSuite{Region: aws.USEast}) + +// AmazonClientSuite tests the client against a live S3 server. +type AmazonClientSuite struct { + aws.Region + srv AmazonServer + ClientTests +} + +func (s *AmazonClientSuite) SetUpSuite(c *C) { + if !testutil.Amazon { + c.Skip("live tests against AWS disabled (no -amazon)") + } + s.srv.SetUp(c) + s.s3 = s3.New(s.srv.auth, s.Region) + // In case tests were interrupted in the middle before. + s.ClientTests.Cleanup() +} + +func (s *AmazonClientSuite) TearDownTest(c *C) { + s.ClientTests.Cleanup() +} + +// AmazonDomainClientSuite tests the client against a live S3 +// server using bucket names in the endpoint domain name rather +// than the request path. +type AmazonDomainClientSuite struct { + aws.Region + srv AmazonServer + ClientTests +} + +func (s *AmazonDomainClientSuite) SetUpSuite(c *C) { + if !testutil.Amazon { + c.Skip("live tests against AWS disabled (no -amazon)") + } + s.srv.SetUp(c) + region := s.Region + region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com" + s.s3 = s3.New(s.srv.auth, region) + s.ClientTests.Cleanup() +} + +func (s *AmazonDomainClientSuite) TearDownTest(c *C) { + s.ClientTests.Cleanup() +} + +// ClientTests defines integration tests designed to test the client. +// It is not used as a test suite in itself, but embedded within +// another type. +type ClientTests struct { + s3 *s3.S3 + authIsBroken bool +} + +func (s *ClientTests) Cleanup() { + killBucket(testBucket(s.s3)) +} + +func testBucket(s *s3.S3) *s3.Bucket { + // Watch out! If this function is corrupted and made to match with something + // people own, killBucket will happily remove *everything* inside the bucket. + key := s.Auth.AccessKey + if len(key) >= 8 { + key = s.Auth.AccessKey[:8] + } + return s.Bucket(fmt.Sprintf("goamz-%s-%s", s.Region.Name, key)) +} + +var attempts = aws.AttemptStrategy{ + Min: 5, + Total: 20 * time.Second, + Delay: 100 * time.Millisecond, +} + +func killBucket(b *s3.Bucket) { + var err error + for attempt := attempts.Start(); attempt.Next(); { + err = b.DelBucket() + if err == nil { + return + } + if _, ok := err.(*net.DNSError); ok { + return + } + e, ok := err.(*s3.Error) + if ok && e.Code == "NoSuchBucket" { + return + } + if ok && e.Code == "BucketNotEmpty" { + // Errors are ignored here. Just retry. + resp, err := b.List("", "", "", 1000) + if err == nil { + for _, key := range resp.Contents { + _ = b.Del(key.Key) + } + } + multis, _, _ := b.ListMulti("", "") + for _, m := range multis { + _ = m.Abort() + } + } + } + message := "cannot delete test bucket" + if err != nil { + message += ": " + err.Error() + } + panic(message) +} + +func get(url string) ([]byte, error) { + for attempt := attempts.Start(); attempt.Next(); { + resp, err := http.Get(url) + if err != nil { + if attempt.HasNext() { + continue + } + return nil, err + } + data, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + if attempt.HasNext() { + continue + } + return nil, err + } + return data, err + } + panic("unreachable") +} + +func (s *ClientTests) TestBasicFunctionality(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.PublicRead) + c.Assert(err, IsNil) + + err = b.Put("name", []byte("yo!"), "text/plain", s3.PublicRead) + c.Assert(err, IsNil) + defer b.Del("name") + + data, err := b.Get("name") + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "yo!") + + data, err = get(b.URL("name")) + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "yo!") + + buf := bytes.NewBufferString("hey!") + err = b.PutReader("name2", buf, int64(buf.Len()), "text/plain", s3.Private) + c.Assert(err, IsNil) + defer b.Del("name2") + + rc, err := b.GetReader("name2") + c.Assert(err, IsNil) + data, err = ioutil.ReadAll(rc) + c.Check(err, IsNil) + c.Check(string(data), Equals, "hey!") + rc.Close() + + data, err = get(b.SignedURL("name2", time.Now().Add(time.Hour))) + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "hey!") + + if !s.authIsBroken { + data, err = get(b.SignedURL("name2", time.Now().Add(-time.Hour))) + c.Assert(err, IsNil) + c.Assert(string(data), Matches, "(?s).*AccessDenied.*") + } + + err = b.DelBucket() + c.Assert(err, NotNil) + + s3err, ok := err.(*s3.Error) + c.Assert(ok, Equals, true) + c.Assert(s3err.Code, Equals, "BucketNotEmpty") + c.Assert(s3err.BucketName, Equals, b.Name) + c.Assert(s3err.Message, Equals, "The bucket you tried to delete is not empty") + + err = b.Del("name") + c.Assert(err, IsNil) + err = b.Del("name2") + c.Assert(err, IsNil) + + err = b.DelBucket() + c.Assert(err, IsNil) +} + +func (s *ClientTests) TestCopy(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.PublicRead) + + err = b.Put("name+1", []byte("yo!"), "text/plain", s3.PublicRead) + c.Assert(err, IsNil) + defer b.Del("name+1") + + err = b.Copy("name+1", "name+2", s3.PublicRead) + c.Assert(err, IsNil) + defer b.Del("name+2") + + data, err := b.Get("name+2") + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "yo!") + + err = b.Del("name+1") + c.Assert(err, IsNil) + err = b.Del("name+2") + c.Assert(err, IsNil) + + err = b.DelBucket() + c.Assert(err, IsNil) +} + +func (s *ClientTests) TestGetNotFound(c *C) { + b := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey) + data, err := b.Get("non-existent") + + s3err, _ := err.(*s3.Error) + c.Assert(s3err, NotNil) + c.Assert(s3err.StatusCode, Equals, 404) + c.Assert(s3err.Code, Equals, "NoSuchBucket") + c.Assert(s3err.Message, Equals, "The specified bucket does not exist") + c.Assert(data, IsNil) +} + +// Communicate with all endpoints to see if they are alive. +func (s *ClientTests) TestRegions(c *C) { + errs := make(chan error, len(aws.Regions)) + for _, region := range aws.Regions { + go func(r aws.Region) { + s := s3.New(s.s3.Auth, r) + b := s.Bucket("goamz-" + s.Auth.AccessKey) + _, err := b.Get("non-existent") + errs <- err + }(region) + } + for _ = range aws.Regions { + err := <-errs + if err != nil { + s3_err, ok := err.(*s3.Error) + if ok { + c.Check(s3_err.Code, Matches, "NoSuchBucket") + } else if _, ok = err.(*net.DNSError); ok { + // Okay as well. + } else { + c.Errorf("Non-S3 error: %s", err) + } + } else { + c.Errorf("Test should have errored but it seems to have succeeded") + } + } +} + +var objectNames = []string{ + "index.html", + "index2.html", + "photos/2006/February/sample2.jpg", + "photos/2006/February/sample3.jpg", + "photos/2006/February/sample4.jpg", + "photos/2006/January/sample.jpg", + "test/bar", + "test/foo", +} + +func keys(names ...string) []s3.Key { + ks := make([]s3.Key, len(names)) + for i, name := range names { + ks[i].Key = name + } + return ks +} + +// As the ListResp specifies all the parameters to the +// request too, we use it to specify request parameters +// and expected results. The Contents field is +// used only for the key names inside it. +var listTests = []s3.ListResp{ + // normal list. + { + Contents: keys(objectNames...), + }, { + Marker: objectNames[0], + Contents: keys(objectNames[1:]...), + }, { + Marker: objectNames[0] + "a", + Contents: keys(objectNames[1:]...), + }, { + Marker: "z", + }, + + // limited results. + { + MaxKeys: 2, + Contents: keys(objectNames[0:2]...), + IsTruncated: true, + }, { + MaxKeys: 2, + Marker: objectNames[0], + Contents: keys(objectNames[1:3]...), + IsTruncated: true, + }, { + MaxKeys: 2, + Marker: objectNames[len(objectNames)-2], + Contents: keys(objectNames[len(objectNames)-1:]...), + }, + + // with delimiter + { + Delimiter: "/", + CommonPrefixes: []string{"photos/", "test/"}, + Contents: keys("index.html", "index2.html"), + }, { + Delimiter: "/", + Prefix: "photos/2006/", + CommonPrefixes: []string{"photos/2006/February/", "photos/2006/January/"}, + }, { + Delimiter: "/", + Prefix: "t", + CommonPrefixes: []string{"test/"}, + }, { + Delimiter: "/", + MaxKeys: 1, + Contents: keys("index.html"), + IsTruncated: true, + }, { + Delimiter: "/", + MaxKeys: 1, + Marker: "index2.html", + CommonPrefixes: []string{"photos/"}, + IsTruncated: true, + }, { + Delimiter: "/", + MaxKeys: 1, + Marker: "photos/", + CommonPrefixes: []string{"test/"}, + IsTruncated: false, + }, { + Delimiter: "Feb", + CommonPrefixes: []string{"photos/2006/Feb"}, + Contents: keys("index.html", "index2.html", "photos/2006/January/sample.jpg", "test/bar", "test/foo"), + }, +} + +func (s *ClientTests) TestDoublePutBucket(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.PublicRead) + c.Assert(err, IsNil) + + err = b.PutBucket(s3.PublicRead) + if err != nil { + c.Assert(err, FitsTypeOf, new(s3.Error)) + c.Assert(err.(*s3.Error).Code, Equals, "BucketAlreadyOwnedByYou") + } +} + +func (s *ClientTests) TestBucketList(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + objData := make(map[string][]byte) + for i, path := range objectNames { + data := []byte(strings.Repeat("a", i)) + err := b.Put(path, data, "text/plain", s3.Private) + c.Assert(err, IsNil) + defer b.Del(path) + objData[path] = data + } + + for i, t := range listTests { + c.Logf("test %d", i) + resp, err := b.List(t.Prefix, t.Delimiter, t.Marker, t.MaxKeys) + c.Assert(err, IsNil) + c.Check(resp.Name, Equals, b.Name) + c.Check(resp.Delimiter, Equals, t.Delimiter) + c.Check(resp.IsTruncated, Equals, t.IsTruncated) + c.Check(resp.CommonPrefixes, DeepEquals, t.CommonPrefixes) + checkContents(c, resp.Contents, objData, t.Contents) + } +} + +func etag(data []byte) string { + sum := md5.New() + sum.Write(data) + return fmt.Sprintf(`"%x"`, sum.Sum(nil)) +} + +func checkContents(c *C, contents []s3.Key, data map[string][]byte, expected []s3.Key) { + c.Assert(contents, HasLen, len(expected)) + for i, k := range contents { + c.Check(k.Key, Equals, expected[i].Key) + // TODO mtime + c.Check(k.Size, Equals, int64(len(data[k.Key]))) + c.Check(k.ETag, Equals, etag(data[k.Key])) + } +} + +func (s *ClientTests) TestMultiInitPutList(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + c.Assert(multi.UploadId, Matches, ".+") + defer multi.Abort() + + var sent []s3.Part + + for i := 0; i < 5; i++ { + p, err := multi.PutPart(i+1, strings.NewReader(fmt.Sprintf("", i+1))) + c.Assert(err, IsNil) + c.Assert(p.N, Equals, i+1) + c.Assert(p.Size, Equals, int64(8)) + c.Assert(p.ETag, Matches, ".+") + sent = append(sent, p) + } + + s3.SetListPartsMax(2) + + parts, err := multi.ListParts() + c.Assert(err, IsNil) + c.Assert(parts, HasLen, len(sent)) + for i := range parts { + c.Assert(parts[i].N, Equals, sent[i].N) + c.Assert(parts[i].Size, Equals, sent[i].Size) + c.Assert(parts[i].ETag, Equals, sent[i].ETag) + } + + err = multi.Complete(parts) + s3err, failed := err.(*s3.Error) + c.Assert(failed, Equals, true) + c.Assert(s3err.Code, Equals, "EntityTooSmall") + + err = multi.Abort() + c.Assert(err, IsNil) + _, err = multi.ListParts() + s3err, ok := err.(*s3.Error) + c.Assert(ok, Equals, true) + c.Assert(s3err.Code, Equals, "NoSuchUpload") +} + +// This may take a minute or more due to the minimum size accepted S3 +// on multipart upload parts. +func (s *ClientTests) TestMultiComplete(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + c.Assert(multi.UploadId, Matches, ".+") + defer multi.Abort() + + // Minimum size S3 accepts for all but the last part is 5MB. + data1 := make([]byte, 5*1024*1024) + data2 := []byte("") + + part1, err := multi.PutPart(1, bytes.NewReader(data1)) + c.Assert(err, IsNil) + part2, err := multi.PutPart(2, bytes.NewReader(data2)) + c.Assert(err, IsNil) + + // Purposefully reversed. The order requirement must be handled. + err = multi.Complete([]s3.Part{part2, part1}) + c.Assert(err, IsNil) + + data, err := b.Get("multi") + c.Assert(err, IsNil) + + c.Assert(len(data), Equals, len(data1)+len(data2)) + for i := range data1 { + if data[i] != data1[i] { + c.Fatalf("uploaded object at byte %d: want %d, got %d", data1[i], data[i]) + } + } + c.Assert(string(data[len(data1):]), Equals, string(data2)) +} + +type multiList []*s3.Multi + +func (l multiList) Len() int { return len(l) } +func (l multiList) Less(i, j int) bool { return l[i].Key < l[j].Key } +func (l multiList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +func (s *ClientTests) TestListMulti(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + // Ensure an empty state before testing its behavior. + multis, _, err := b.ListMulti("", "") + for _, m := range multis { + err := m.Abort() + c.Assert(err, IsNil) + } + + keys := []string{ + "a/multi2", + "a/multi3", + "b/multi4", + "multi1", + } + for _, key := range keys { + m, err := b.InitMulti(key, "", s3.Private) + c.Assert(err, IsNil) + defer m.Abort() + } + + // Amazon's implementation of the multiple-request listing for + // multipart uploads in progress seems broken in multiple ways. + // (next tokens are not provided, etc). + //s3.SetListMultiMax(2) + + multis, prefixes, err := b.ListMulti("", "") + c.Assert(err, IsNil) + for attempt := attempts.Start(); attempt.Next() && len(multis) < len(keys); { + multis, prefixes, err = b.ListMulti("", "") + c.Assert(err, IsNil) + } + sort.Sort(multiList(multis)) + c.Assert(prefixes, IsNil) + var gotKeys []string + for _, m := range multis { + gotKeys = append(gotKeys, m.Key) + } + c.Assert(gotKeys, DeepEquals, keys) + for _, m := range multis { + c.Assert(m.Bucket, Equals, b) + c.Assert(m.UploadId, Matches, ".+") + } + + multis, prefixes, err = b.ListMulti("", "/") + for attempt := attempts.Start(); attempt.Next() && len(prefixes) < 2; { + multis, prefixes, err = b.ListMulti("", "") + c.Assert(err, IsNil) + } + c.Assert(err, IsNil) + c.Assert(prefixes, DeepEquals, []string{"a/", "b/"}) + c.Assert(multis, HasLen, 1) + c.Assert(multis[0].Bucket, Equals, b) + c.Assert(multis[0].Key, Equals, "multi1") + c.Assert(multis[0].UploadId, Matches, ".+") + + for attempt := attempts.Start(); attempt.Next() && len(multis) < 2; { + multis, prefixes, err = b.ListMulti("", "") + c.Assert(err, IsNil) + } + multis, prefixes, err = b.ListMulti("a/", "/") + c.Assert(err, IsNil) + c.Assert(prefixes, IsNil) + c.Assert(multis, HasLen, 2) + c.Assert(multis[0].Bucket, Equals, b) + c.Assert(multis[0].Key, Equals, "a/multi2") + c.Assert(multis[0].UploadId, Matches, ".+") + c.Assert(multis[1].Bucket, Equals, b) + c.Assert(multis[1].Key, Equals, "a/multi3") + c.Assert(multis[1].UploadId, Matches, ".+") +} + +func (s *ClientTests) TestMultiPutAllZeroLength(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.Private) + c.Assert(err, IsNil) + + multi, err := b.InitMulti("multi", "text/plain", s3.Private) + c.Assert(err, IsNil) + defer multi.Abort() + + // This tests an edge case. Amazon requires at least one + // part for multiprat uploads to work, even the part is empty. + parts, err := multi.PutAll(strings.NewReader(""), 5*1024*1024) + c.Assert(err, IsNil) + c.Assert(parts, HasLen, 1) + c.Assert(parts[0].Size, Equals, int64(0)) + c.Assert(parts[0].ETag, Equals, `"d41d8cd98f00b204e9800998ecf8427e"`) + + err = multi.Complete(parts) + c.Assert(err, IsNil) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go new file mode 100644 index 000000000..d3d49966e --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go @@ -0,0 +1,79 @@ +package s3_test + +import ( + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/s3" + "github.com/mitchellh/goamz/s3/s3test" + . "github.com/motain/gocheck" +) + +type LocalServer struct { + auth aws.Auth + region aws.Region + srv *s3test.Server + config *s3test.Config +} + +func (s *LocalServer) SetUp(c *C) { + srv, err := s3test.NewServer(s.config) + c.Assert(err, IsNil) + c.Assert(srv, NotNil) + + s.srv = srv + s.region = aws.Region{ + Name: "faux-region-1", + S3Endpoint: srv.URL(), + S3LocationConstraint: true, // s3test server requires a LocationConstraint + } +} + +// LocalServerSuite defines tests that will run +// against the local s3test server. It includes +// selected tests from ClientTests; +// when the s3test functionality is sufficient, it should +// include all of them, and ClientTests can be simply embedded. +type LocalServerSuite struct { + srv LocalServer + clientTests ClientTests +} + +var ( + // run tests twice, once in us-east-1 mode, once not. + _ = Suite(&LocalServerSuite{}) + _ = Suite(&LocalServerSuite{ + srv: LocalServer{ + config: &s3test.Config{ + Send409Conflict: true, + }, + }, + }) +) + +func (s *LocalServerSuite) SetUpSuite(c *C) { + s.srv.SetUp(c) + s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region) + + // TODO Sadly the fake server ignores auth completely right now. :-( + s.clientTests.authIsBroken = true + s.clientTests.Cleanup() +} + +func (s *LocalServerSuite) TearDownTest(c *C) { + s.clientTests.Cleanup() +} + +func (s *LocalServerSuite) TestBasicFunctionality(c *C) { + s.clientTests.TestBasicFunctionality(c) +} + +func (s *LocalServerSuite) TestGetNotFound(c *C) { + s.clientTests.TestGetNotFound(c) +} + +func (s *LocalServerSuite) TestBucketList(c *C) { + s.clientTests.TestBucketList(c) +} + +func (s *LocalServerSuite) TestDoublePutBucket(c *C) { + s.clientTests.TestDoublePutBucket(c) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go new file mode 100644 index 000000000..40e915e94 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go @@ -0,0 +1,666 @@ +package s3test + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "encoding/xml" + "fmt" + "github.com/mitchellh/goamz/s3" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const debug = false + +type s3Error struct { + statusCode int + XMLName struct{} `xml:"Error"` + Code string + Message string + BucketName string + RequestId string + HostId string +} + +type action struct { + srv *Server + w http.ResponseWriter + req *http.Request + reqId string +} + +// Config controls the internal behaviour of the Server. A nil config is the default +// and behaves as if all configurations assume their default behaviour. Once passed +// to NewServer, the configuration must not be modified. +type Config struct { + // Send409Conflict controls how the Server will respond to calls to PUT on a + // previously existing bucket. The default is false, and corresponds to the + // us-east-1 s3 enpoint. Setting this value to true emulates the behaviour of + // all other regions. + // http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html + Send409Conflict bool +} + +func (c *Config) send409Conflict() bool { + if c != nil { + return c.Send409Conflict + } + return false +} + +// Server is a fake S3 server for testing purposes. +// All of the data for the server is kept in memory. +type Server struct { + url string + reqId int + listener net.Listener + mu sync.Mutex + buckets map[string]*bucket + config *Config +} + +type bucket struct { + name string + acl s3.ACL + ctime time.Time + objects map[string]*object +} + +type object struct { + name string + mtime time.Time + meta http.Header // metadata to return with requests. + checksum []byte // also held as Content-MD5 in meta. + data []byte +} + +// A resource encapsulates the subject of an HTTP request. +// The resource referred to may or may not exist +// when the request is made. +type resource interface { + put(a *action) interface{} + get(a *action) interface{} + post(a *action) interface{} + delete(a *action) interface{} +} + +func NewServer(config *Config) (*Server, error) { + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + return nil, fmt.Errorf("cannot listen on localhost: %v", err) + } + srv := &Server{ + listener: l, + url: "http://" + l.Addr().String(), + buckets: make(map[string]*bucket), + config: config, + } + go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + srv.serveHTTP(w, req) + })) + return srv, nil +} + +// Quit closes down the server. +func (srv *Server) Quit() { + srv.listener.Close() +} + +// URL returns a URL for the server. +func (srv *Server) URL() string { + return srv.url +} + +func fatalf(code int, codeStr string, errf string, a ...interface{}) { + panic(&s3Error{ + statusCode: code, + Code: codeStr, + Message: fmt.Sprintf(errf, a...), + }) +} + +// serveHTTP serves the S3 protocol. +func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) { + // ignore error from ParseForm as it's usually spurious. + req.ParseForm() + + srv.mu.Lock() + defer srv.mu.Unlock() + + if debug { + log.Printf("s3test %q %q", req.Method, req.URL) + } + a := &action{ + srv: srv, + w: w, + req: req, + reqId: fmt.Sprintf("%09X", srv.reqId), + } + srv.reqId++ + + var r resource + defer func() { + switch err := recover().(type) { + case *s3Error: + switch r := r.(type) { + case objectResource: + err.BucketName = r.bucket.name + case bucketResource: + err.BucketName = r.name + } + err.RequestId = a.reqId + // TODO HostId + w.Header().Set("Content-Type", `xml version="1.0" encoding="UTF-8"`) + w.WriteHeader(err.statusCode) + xmlMarshal(w, err) + case nil: + default: + panic(err) + } + }() + + r = srv.resourceForURL(req.URL) + + var resp interface{} + switch req.Method { + case "PUT": + resp = r.put(a) + case "GET", "HEAD": + resp = r.get(a) + case "DELETE": + resp = r.delete(a) + case "POST": + resp = r.post(a) + default: + fatalf(400, "MethodNotAllowed", "unknown http request method %q", req.Method) + } + if resp != nil && req.Method != "HEAD" { + xmlMarshal(w, resp) + } +} + +// xmlMarshal is the same as xml.Marshal except that +// it panics on error. The marshalling should not fail, +// but we want to know if it does. +func xmlMarshal(w io.Writer, x interface{}) { + if err := xml.NewEncoder(w).Encode(x); err != nil { + panic(fmt.Errorf("error marshalling %#v: %v", x, err)) + } +} + +// In a fully implemented test server, each of these would have +// its own resource type. +var unimplementedBucketResourceNames = map[string]bool{ + "acl": true, + "lifecycle": true, + "policy": true, + "location": true, + "logging": true, + "notification": true, + "versions": true, + "requestPayment": true, + "versioning": true, + "website": true, + "uploads": true, +} + +var unimplementedObjectResourceNames = map[string]bool{ + "uploadId": true, + "acl": true, + "torrent": true, + "uploads": true, +} + +var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?") + +// resourceForURL returns a resource object for the given URL. +func (srv *Server) resourceForURL(u *url.URL) (r resource) { + + if u.Path == "/" { + return serviceResource{ + buckets: srv.buckets, + } + } + + m := pathRegexp.FindStringSubmatch(u.Path) + if m == nil { + fatalf(404, "InvalidURI", "Couldn't parse the specified URI") + } + bucketName := m[2] + objectName := m[4] + if bucketName == "" { + return nullResource{} // root + } + b := bucketResource{ + name: bucketName, + bucket: srv.buckets[bucketName], + } + q := u.Query() + if objectName == "" { + for name := range q { + if unimplementedBucketResourceNames[name] { + return nullResource{} + } + } + return b + + } + if b.bucket == nil { + fatalf(404, "NoSuchBucket", "The specified bucket does not exist") + } + objr := objectResource{ + name: objectName, + version: q.Get("versionId"), + bucket: b.bucket, + } + for name := range q { + if unimplementedObjectResourceNames[name] { + return nullResource{} + } + } + if obj := objr.bucket.objects[objr.name]; obj != nil { + objr.object = obj + } + return objr +} + +// nullResource has error stubs for all resource methods. +type nullResource struct{} + +func notAllowed() interface{} { + fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource") + return nil +} + +func (nullResource) put(a *action) interface{} { return notAllowed() } +func (nullResource) get(a *action) interface{} { return notAllowed() } +func (nullResource) post(a *action) interface{} { return notAllowed() } +func (nullResource) delete(a *action) interface{} { return notAllowed() } + +const timeFormat = "2006-01-02T15:04:05.000Z07:00" + +type serviceResource struct { + buckets map[string]*bucket +} + +func (serviceResource) put(a *action) interface{} { return notAllowed() } +func (serviceResource) post(a *action) interface{} { return notAllowed() } +func (serviceResource) delete(a *action) interface{} { return notAllowed() } + +// GET on an s3 service lists the buckets. +// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html +func (r serviceResource) get(a *action) interface{} { + type respBucket struct { + Name string + } + + type response struct { + Buckets []respBucket `xml:">Bucket"` + } + + resp := response{} + + for _, bucketPtr := range r.buckets { + bkt := respBucket{ + Name: bucketPtr.name, + } + resp.Buckets = append(resp.Buckets, bkt) + } + + return &resp +} + +type bucketResource struct { + name string + bucket *bucket // non-nil if the bucket already exists. +} + +// GET on a bucket lists the objects in the bucket. +// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketGET.html +func (r bucketResource) get(a *action) interface{} { + if r.bucket == nil { + fatalf(404, "NoSuchBucket", "The specified bucket does not exist") + } + delimiter := a.req.Form.Get("delimiter") + marker := a.req.Form.Get("marker") + maxKeys := -1 + if s := a.req.Form.Get("max-keys"); s != "" { + i, err := strconv.Atoi(s) + if err != nil || i < 0 { + fatalf(400, "invalid value for max-keys: %q", s) + } + maxKeys = i + } + prefix := a.req.Form.Get("prefix") + a.w.Header().Set("Content-Type", "application/xml") + + if a.req.Method == "HEAD" { + return nil + } + + var objs orderedObjects + + // first get all matching objects and arrange them in alphabetical order. + for name, obj := range r.bucket.objects { + if strings.HasPrefix(name, prefix) { + objs = append(objs, obj) + } + } + sort.Sort(objs) + + if maxKeys <= 0 { + maxKeys = 1000 + } + resp := &s3.ListResp{ + Name: r.bucket.name, + Prefix: prefix, + Delimiter: delimiter, + Marker: marker, + MaxKeys: maxKeys, + } + + var prefixes []string + for _, obj := range objs { + if !strings.HasPrefix(obj.name, prefix) { + continue + } + name := obj.name + isPrefix := false + if delimiter != "" { + if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 { + name = obj.name[:len(prefix)+i+len(delimiter)] + if prefixes != nil && prefixes[len(prefixes)-1] == name { + continue + } + isPrefix = true + } + } + if name <= marker { + continue + } + if len(resp.Contents)+len(prefixes) >= maxKeys { + resp.IsTruncated = true + break + } + if isPrefix { + prefixes = append(prefixes, name) + } else { + // Contents contains only keys not found in CommonPrefixes + resp.Contents = append(resp.Contents, obj.s3Key()) + } + } + resp.CommonPrefixes = prefixes + return resp +} + +// orderedObjects holds a slice of objects that can be sorted +// by name. +type orderedObjects []*object + +func (s orderedObjects) Len() int { + return len(s) +} +func (s orderedObjects) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s orderedObjects) Less(i, j int) bool { + return s[i].name < s[j].name +} + +func (obj *object) s3Key() s3.Key { + return s3.Key{ + Key: obj.name, + LastModified: obj.mtime.Format(timeFormat), + Size: int64(len(obj.data)), + ETag: fmt.Sprintf(`"%x"`, obj.checksum), + // TODO StorageClass + // TODO Owner + } +} + +// DELETE on a bucket deletes the bucket if it's not empty. +func (r bucketResource) delete(a *action) interface{} { + b := r.bucket + if b == nil { + fatalf(404, "NoSuchBucket", "The specified bucket does not exist") + } + if len(b.objects) > 0 { + fatalf(400, "BucketNotEmpty", "The bucket you tried to delete is not empty") + } + delete(a.srv.buckets, b.name) + return nil +} + +// PUT on a bucket creates the bucket. +// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketPUT.html +func (r bucketResource) put(a *action) interface{} { + var created bool + if r.bucket == nil { + if !validBucketName(r.name) { + fatalf(400, "InvalidBucketName", "The specified bucket is not valid") + } + if loc := locationConstraint(a); loc == "" { + fatalf(400, "InvalidRequets", "The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.") + } + // TODO validate acl + r.bucket = &bucket{ + name: r.name, + // TODO default acl + objects: make(map[string]*object), + } + a.srv.buckets[r.name] = r.bucket + created = true + } + if !created && a.srv.config.send409Conflict() { + fatalf(409, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.") + } + r.bucket.acl = s3.ACL(a.req.Header.Get("x-amz-acl")) + return nil +} + +func (bucketResource) post(a *action) interface{} { + fatalf(400, "Method", "bucket POST method not available") + return nil +} + +// validBucketName returns whether name is a valid bucket name. +// Here are the rules, from: +// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html +// +// Can contain lowercase letters, numbers, periods (.), underscores (_), +// and dashes (-). You can use uppercase letters for buckets only in the +// US Standard region. +// +// Must start with a number or letter +// +// Must be between 3 and 255 characters long +// +// There's one extra rule (Must not be formatted as an IP address (e.g., 192.168.5.4) +// but the real S3 server does not seem to check that rule, so we will not +// check it either. +// +func validBucketName(name string) bool { + if len(name) < 3 || len(name) > 255 { + return false + } + r := name[0] + if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z') { + return false + } + for _, r := range name { + switch { + case r >= '0' && r <= '9': + case r >= 'a' && r <= 'z': + case r == '_' || r == '-': + case r == '.': + default: + return false + } + } + return true +} + +var responseParams = map[string]bool{ + "content-type": true, + "content-language": true, + "expires": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, +} + +type objectResource struct { + name string + version string + bucket *bucket // always non-nil. + object *object // may be nil. +} + +// GET on an object gets the contents of the object. +// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGET.html +func (objr objectResource) get(a *action) interface{} { + obj := objr.object + if obj == nil { + fatalf(404, "NoSuchKey", "The specified key does not exist.") + } + h := a.w.Header() + // add metadata + for name, d := range obj.meta { + h[name] = d + } + // override header values in response to request parameters. + for name, vals := range a.req.Form { + if strings.HasPrefix(name, "response-") { + name = name[len("response-"):] + if !responseParams[name] { + continue + } + h.Set(name, vals[0]) + } + } + if r := a.req.Header.Get("Range"); r != "" { + fatalf(400, "NotImplemented", "range unimplemented") + } + // TODO Last-Modified-Since + // TODO If-Modified-Since + // TODO If-Unmodified-Since + // TODO If-Match + // TODO If-None-Match + // TODO Connection: close ?? + // TODO x-amz-request-id + h.Set("Content-Length", fmt.Sprint(len(obj.data))) + h.Set("ETag", hex.EncodeToString(obj.checksum)) + h.Set("Last-Modified", obj.mtime.Format(time.RFC1123)) + if a.req.Method == "HEAD" { + return nil + } + // TODO avoid holding the lock when writing data. + _, err := a.w.Write(obj.data) + if err != nil { + // we can't do much except just log the fact. + log.Printf("error writing data: %v", err) + } + return nil +} + +var metaHeaders = map[string]bool{ + "Content-MD5": true, + "x-amz-acl": true, + "Content-Type": true, + "Content-Encoding": true, + "Content-Disposition": true, +} + +// PUT on an object creates the object. +func (objr objectResource) put(a *action) interface{} { + // TODO Cache-Control header + // TODO Expires header + // TODO x-amz-server-side-encryption + // TODO x-amz-storage-class + + // TODO is this correct, or should we erase all previous metadata? + obj := objr.object + if obj == nil { + obj = &object{ + name: objr.name, + meta: make(http.Header), + } + } + + var expectHash []byte + if c := a.req.Header.Get("Content-MD5"); c != "" { + var err error + expectHash, err = hex.DecodeString(c) + if err != nil || len(expectHash) != md5.Size { + fatalf(400, "InvalidDigest", "The Content-MD5 you specified was invalid") + } + } + sum := md5.New() + // TODO avoid holding lock while reading data. + data, err := ioutil.ReadAll(io.TeeReader(a.req.Body, sum)) + if err != nil { + fatalf(400, "TODO", "read error") + } + gotHash := sum.Sum(nil) + if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 { + fatalf(400, "BadDigest", "The Content-MD5 you specified did not match what we received") + } + if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength { + fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header") + } + + // PUT request has been successful - save data and metadata + for key, values := range a.req.Header { + key = http.CanonicalHeaderKey(key) + if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") { + obj.meta[key] = values + } + } + obj.data = data + obj.checksum = gotHash + obj.mtime = time.Now() + objr.bucket.objects[objr.name] = obj + return nil +} + +func (objr objectResource) delete(a *action) interface{} { + delete(objr.bucket.objects, objr.name) + return nil +} + +func (objr objectResource) post(a *action) interface{} { + fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource") + return nil +} + +type CreateBucketConfiguration struct { + LocationConstraint string +} + +// locationConstraint parses the request body (if present). +// If there is no body, an empty string will be returned. +func locationConstraint(a *action) string { + var body bytes.Buffer + if _, err := io.Copy(&body, a.req.Body); err != nil { + fatalf(400, "InvalidRequest", err.Error()) + } + if body.Len() == 0 { + return "" + } + var loc CreateBucketConfiguration + if err := xml.NewDecoder(&body).Decode(&loc); err != nil { + fatalf(400, "InvalidRequest", err.Error()) + } + return loc.LocationConstraint +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go new file mode 100644 index 000000000..262503dc3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go @@ -0,0 +1,126 @@ +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "log" + "sort" + "strings" + + "github.com/mitchellh/goamz/aws" +) + +var b64 = base64.StdEncoding + +// ---------------------------------------------------------------------------- +// S3 signing (http://goo.gl/G1LrK) + +var s3ParamsToSign = map[string]bool{ + "acl": true, + "delete": true, + "location": true, + "logging": true, + "notification": true, + "partNumber": true, + "policy": true, + "requestPayment": true, + "torrent": true, + "uploadId": true, + "uploads": true, + "versionId": true, + "versioning": true, + "versions": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, +} + +func sign(auth aws.Auth, method, canonicalPath string, params, headers map[string][]string) { + var md5, ctype, date, xamz string + var xamzDate bool + var sarray []string + + // add security token + if auth.Token != "" { + headers["x-amz-security-token"] = []string{auth.Token} + } + + if auth.SecretKey == "" { + // no auth secret; skip signing, e.g. for public read-only buckets. + return + } + + for k, v := range headers { + k = strings.ToLower(k) + switch k { + case "content-md5": + md5 = v[0] + case "content-type": + ctype = v[0] + case "date": + if !xamzDate { + date = v[0] + } + default: + if strings.HasPrefix(k, "x-amz-") { + vall := strings.Join(v, ",") + sarray = append(sarray, k+":"+vall) + if k == "x-amz-date" { + xamzDate = true + date = "" + } + } + } + } + if len(sarray) > 0 { + sort.StringSlice(sarray).Sort() + xamz = strings.Join(sarray, "\n") + "\n" + } + + expires := false + if v, ok := params["Expires"]; ok { + // Query string request authentication alternative. + expires = true + date = v[0] + params["AWSAccessKeyId"] = []string{auth.AccessKey} + } + + sarray = sarray[0:0] + for k, v := range params { + if s3ParamsToSign[k] { + for _, vi := range v { + if vi == "" { + sarray = append(sarray, k) + } else { + // "When signing you do not encode these values." + sarray = append(sarray, k+"="+vi) + } + } + } + } + if len(sarray) > 0 { + sort.StringSlice(sarray).Sort() + canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&") + } + + payload := method + "\n" + md5 + "\n" + ctype + "\n" + date + "\n" + xamz + canonicalPath + hash := hmac.New(sha1.New, []byte(auth.SecretKey)) + hash.Write([]byte(payload)) + signature := make([]byte, b64.EncodedLen(hash.Size())) + b64.Encode(signature, hash.Sum(nil)) + + if expires { + params["Signature"] = []string{string(signature)} + } else { + headers["Authorization"] = []string{"AWS " + auth.AccessKey + ":" + string(signature)} + } + + if debug { + log.Printf("Signature payload: %q", payload) + log.Printf("Signature: %q", signature) + } +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go new file mode 100644 index 000000000..9d9fe8d2e --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go @@ -0,0 +1,194 @@ +package s3_test + +import ( + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/s3" + . "github.com/motain/gocheck" +) + +// S3 ReST authentication docs: http://goo.gl/G1LrK + +var testAuth = aws.Auth{"0PN5J17HBGZHT7JJ3X82", "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", ""} +var emptyAuth = aws.Auth{"", "", ""} + +func (s *S) TestSignExampleObjectGet(c *C) { + method := "GET" + path := "/johnsmith/photos/puppy.jpg" + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"}, + } + s3.Sign(testAuth, method, path, nil, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleObjectGetNoAuth(c *C) { + method := "GET" + path := "/johnsmith/photos/puppy.jpg" + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"}, + } + s3.Sign(emptyAuth, method, path, nil, headers) + c.Assert(headers["Authorization"], IsNil) +} + +func (s *S) TestSignExampleObjectPut(c *C) { + method := "PUT" + path := "/johnsmith/photos/puppy.jpg" + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 21:15:45 +0000"}, + "Content-Type": {"image/jpeg"}, + "Content-Length": {"94328"}, + } + s3.Sign(testAuth, method, path, nil, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleList(c *C) { + method := "GET" + path := "/johnsmith/" + params := map[string][]string{ + "prefix": {"photos"}, + "max-keys": {"50"}, + "marker": {"puppy"}, + } + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"}, + "User-Agent": {"Mozilla/5.0"}, + } + s3.Sign(testAuth, method, path, params, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleListNoAuth(c *C) { + method := "GET" + path := "/johnsmith/" + params := map[string][]string{ + "prefix": {"photos"}, + "max-keys": {"50"}, + "marker": {"puppy"}, + } + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"}, + "User-Agent": {"Mozilla/5.0"}, + } + s3.Sign(emptyAuth, method, path, params, headers) + c.Assert(headers["Authorization"], IsNil) +} + +func (s *S) TestSignExampleFetch(c *C) { + method := "GET" + path := "/johnsmith/" + params := map[string][]string{ + "acl": {""}, + } + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"}, + } + s3.Sign(testAuth, method, path, params, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleFetchNoAuth(c *C) { + method := "GET" + path := "/johnsmith/" + params := map[string][]string{ + "acl": {""}, + } + headers := map[string][]string{ + "Host": {"johnsmith.s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"}, + } + s3.Sign(emptyAuth, method, path, params, headers) + c.Assert(headers["Authorization"], IsNil) +} + +func (s *S) TestSignExampleDelete(c *C) { + method := "DELETE" + path := "/johnsmith/photos/puppy.jpg" + params := map[string][]string{} + headers := map[string][]string{ + "Host": {"s3.amazonaws.com"}, + "Date": {"Tue, 27 Mar 2007 21:20:27 +0000"}, + "User-Agent": {"dotnet"}, + "x-amz-date": {"Tue, 27 Mar 2007 21:20:26 +0000"}, + } + s3.Sign(testAuth, method, path, params, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleUpload(c *C) { + method := "PUT" + path := "/static.johnsmith.net/db-backup.dat.gz" + params := map[string][]string{} + headers := map[string][]string{ + "Host": {"static.johnsmith.net:8080"}, + "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"}, + "User-Agent": {"curl/7.15.5"}, + "x-amz-acl": {"public-read"}, + "content-type": {"application/x-download"}, + "Content-MD5": {"4gJE4saaMU4BqNR0kLY+lw=="}, + "X-Amz-Meta-ReviewedBy": {"joe@johnsmith.net,jane@johnsmith.net"}, + "X-Amz-Meta-FileChecksum": {"0x02661779"}, + "X-Amz-Meta-ChecksumAlgorithm": {"crc32"}, + "Content-Disposition": {"attachment; filename=database.dat"}, + "Content-Encoding": {"gzip"}, + "Content-Length": {"5913339"}, + } + s3.Sign(testAuth, method, path, params, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleListAllMyBuckets(c *C) { + method := "GET" + path := "/" + headers := map[string][]string{ + "Host": {"s3.amazonaws.com"}, + "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"}, + } + s3.Sign(testAuth, method, path, nil, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:Db+gepJSUbZKwpx1FR0DLtEYoZA=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +func (s *S) TestSignExampleUnicodeKeys(c *C) { + method := "GET" + path := "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re" + headers := map[string][]string{ + "Host": {"s3.amazonaws.com"}, + "Date": {"Wed, 28 Mar 2007 01:49:49 +0000"}, + } + s3.Sign(testAuth, method, path, nil, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:dxhSBHoI6eVSPcXJqEghlUzZMnY=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) +} + +// Not included in AWS documentation + +func (s *S) TestSignWithIAMToken(c *C) { + method := "GET" + path := "/" + headers := map[string][]string{ + "Host": {"s3.amazonaws.com"}, + "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"}, + } + + authWithToken := testAuth + authWithToken.Token = "totallysecret" + + s3.Sign(authWithToken, method, path, nil, headers) + expected := "AWS 0PN5J17HBGZHT7JJ3X82:SJ0yQO7NpHyXJ7zkxY+/fGQ6aUw=" + c.Assert(headers["Authorization"], DeepEquals, []string{expected}) + c.Assert(headers["x-amz-security-token"], DeepEquals, []string{authWithToken.Token}) +} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE b/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE new file mode 100644 index 000000000..968b45384 --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE @@ -0,0 +1,14 @@ +Copyright (c) 2013 Vaughan Newton + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md b/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md new file mode 100644 index 000000000..d5cd4e74b --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md @@ -0,0 +1,70 @@ +go-ini +====== + +INI parsing library for Go (golang). + +View the API documentation [here](http://godoc.org/github.com/vaughan0/go-ini). + +Usage +----- + +Parse an INI file: + +```go +import "github.com/vaughan0/go-ini" + +file, err := ini.LoadFile("myfile.ini") +``` + +Get data from the parsed file: + +```go +name, ok := file.Get("person", "name") +if !ok { + panic("'name' variable missing from 'person' section") +} +``` + +Iterate through values in a section: + +```go +for key, value := range file["mysection"] { + fmt.Printf("%s => %s\n", key, value) +} +``` + +Iterate through sections in a file: + +```go +for name, section := range file { + fmt.Printf("Section name: %s\n", name) +} +``` + +File Format +----------- + +INI files are parsed by go-ini line-by-line. Each line may be one of the following: + + * A section definition: [section-name] + * A property: key = value + * A comment: #blahblah _or_ ;blahblah + * Blank. The line will be ignored. + +Properties defined before any section headers are placed in the default section, which has +the empty string as it's key. + +Example: + +```ini +# I am a comment +; So am I! + +[apples] +colour = red or green +shape = applish + +[oranges] +shape = square +colour = blue +``` diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go new file mode 100644 index 000000000..81aeb32f8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go @@ -0,0 +1,123 @@ +// Package ini provides functions for parsing INI configuration files. +package ini + +import ( + "bufio" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +var ( + sectionRegex = regexp.MustCompile(`^\[(.*)\]$`) + assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) +) + +// ErrSyntax is returned when there is a syntax error in an INI file. +type ErrSyntax struct { + Line int + Source string // The contents of the erroneous line, without leading or trailing whitespace +} + +func (e ErrSyntax) Error() string { + return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source) +} + +// A File represents a parsed INI file. +type File map[string]Section + +// A Section represents a single section of an INI file. +type Section map[string]string + +// Returns a named Section. A Section will be created if one does not already exist for the given name. +func (f File) Section(name string) Section { + section := f[name] + if section == nil { + section = make(Section) + f[name] = section + } + return section +} + +// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup. +func (f File) Get(section, key string) (value string, ok bool) { + if s := f[section]; s != nil { + value, ok = s[key] + } + return +} + +// Loads INI data from a reader and stores the data in the File. +func (f File) Load(in io.Reader) (err error) { + bufin, ok := in.(*bufio.Reader) + if !ok { + bufin = bufio.NewReader(in) + } + return parseFile(bufin, f) +} + +// Loads INI data from a named file and stores the data in the File. +func (f File) LoadFile(file string) (err error) { + in, err := os.Open(file) + if err != nil { + return + } + defer in.Close() + return f.Load(in) +} + +func parseFile(in *bufio.Reader, file File) (err error) { + section := "" + lineNum := 0 + for done := false; !done; { + var line string + if line, err = in.ReadString('\n'); err != nil { + if err == io.EOF { + done = true + } else { + return + } + } + lineNum++ + line = strings.TrimSpace(line) + if len(line) == 0 { + // Skip blank lines + continue + } + if line[0] == ';' || line[0] == '#' { + // Skip comments + continue + } + + if groups := assignRegex.FindStringSubmatch(line); groups != nil { + key, val := groups[1], groups[2] + key, val = strings.TrimSpace(key), strings.TrimSpace(val) + file.Section(section)[key] = val + } else if groups := sectionRegex.FindStringSubmatch(line); groups != nil { + name := strings.TrimSpace(groups[1]) + section = name + // Create the section if it does not exist + file.Section(section) + } else { + return ErrSyntax{lineNum, line} + } + + } + return nil +} + +// Loads and returns a File from a reader. +func Load(in io.Reader) (File, error) { + file := make(File) + err := file.Load(in) + return file, err +} + +// Loads and returns an INI File from a file on disk. +func LoadFile(filename string) (File, error) { + file := make(File) + err := file.LoadFile(filename) + return file, err +} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go new file mode 100644 index 000000000..38a6f0004 --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go @@ -0,0 +1,43 @@ +package ini + +import ( + "reflect" + "syscall" + "testing" +) + +func TestLoadFile(t *testing.T) { + originalOpenFiles := numFilesOpen(t) + + file, err := LoadFile("test.ini") + if err != nil { + t.Fatal(err) + } + + if originalOpenFiles != numFilesOpen(t) { + t.Error("test.ini not closed") + } + + if !reflect.DeepEqual(file, File{"default": {"stuff": "things"}}) { + t.Error("file not read correctly") + } +} + +func numFilesOpen(t *testing.T) (num uint64) { + var rlimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + t.Fatal(err) + } + maxFds := int(rlimit.Cur) + + var stat syscall.Stat_t + for i := 0; i < maxFds; i++ { + if syscall.Fstat(i, &stat) == nil { + num++ + } else { + return + } + } + return +} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go new file mode 100644 index 000000000..06a4d05ea --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go @@ -0,0 +1,89 @@ +package ini + +import ( + "reflect" + "strings" + "testing" +) + +func TestLoad(t *testing.T) { + src := ` + # Comments are ignored + + herp = derp + + [foo] + hello=world + whitespace should = not matter + ; sneaky semicolon-style comment + multiple = equals = signs + + [bar] + this = that` + + file, err := Load(strings.NewReader(src)) + if err != nil { + t.Fatal(err) + } + check := func(section, key, expect string) { + if value, _ := file.Get(section, key); value != expect { + t.Errorf("Get(%q, %q): expected %q, got %q", section, key, expect, value) + } + } + + check("", "herp", "derp") + check("foo", "hello", "world") + check("foo", "whitespace should", "not matter") + check("foo", "multiple", "equals = signs") + check("bar", "this", "that") +} + +func TestSyntaxError(t *testing.T) { + src := ` + # Line 2 + [foo] + bar = baz + # Here's an error on line 6: + wut? + herp = derp` + _, err := Load(strings.NewReader(src)) + t.Logf("%T: %v", err, err) + if err == nil { + t.Fatal("expected an error, got nil") + } + syntaxErr, ok := err.(ErrSyntax) + if !ok { + t.Fatal("expected an error of type ErrSyntax") + } + if syntaxErr.Line != 6 { + t.Fatal("incorrect line number") + } + if syntaxErr.Source != "wut?" { + t.Fatal("incorrect source") + } +} + +func TestDefinedSectionBehaviour(t *testing.T) { + check := func(src string, expect File) { + file, err := Load(strings.NewReader(src)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(file, expect) { + t.Errorf("expected %v, got %v", expect, file) + } + } + // No sections for an empty file + check("", File{}) + // Default section only if there are actually values for it + check("foo=bar", File{"": {"foo": "bar"}}) + // User-defined sections should always be present, even if empty + check("[a]\n[b]\nfoo=bar", File{ + "a": {}, + "b": {"foo": "bar"}, + }) + check("foo=bar\n[a]\nthis=that", File{ + "": {"foo": "bar"}, + "a": {"this": "that"}, + }) +} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini b/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini new file mode 100644 index 000000000..d13c999e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini @@ -0,0 +1,2 @@ +[default] +stuff = things From d1554d11338b0c62aba426823b7f145174404af1 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Fri, 15 May 2015 16:53:00 -0500 Subject: [PATCH 04/13] Add backend test for s3 --- backend/s3_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 backend/s3_test.go diff --git a/backend/s3_test.go b/backend/s3_test.go new file mode 100644 index 000000000..84b4d6d74 --- /dev/null +++ b/backend/s3_test.go @@ -0,0 +1,43 @@ +package backend_test + +import ( + "testing" + + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/s3" + "github.com/mitchellh/goamz/testutil" + + bes3 "github.com/restic/restic/backend/s3" + . "github.com/restic/restic/test" +) + +var testServer = testutil.NewHTTPServer() + +func setupS3Backend(t *testing.T) *bes3.S3 { + testServer.Start() + auth := aws.Auth{"abc", "123", ""} + service := s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL}) + bucket := service.Bucket("testbucket") + err := bucket.PutBucket("private") + OK(t, err) + + t.Logf("created s3 backend locally at %s", testServer.URL) + + return bes3.S3{bucket: bucket, path: "testbucket"} +} + +func teardownS3Backend(t *testing.T, b *bes3.S3) { + if !*TestCleanup { + t.Logf("leaving backend at %s\n", b.Location()) + return + } + + testServer.Flush() +} + +func TestS3Backend(t *testing.T) { + s := setupS3Backend(t) + defer teardownS3Backend(t, s) + + testBackend(s, t) +} From b793261ba15c79070c0c2dcf8fd79a158fc883d2 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Fri, 15 May 2015 17:03:34 -0500 Subject: [PATCH 05/13] Add dependencies for s3 tests. --- Godeps/Godeps.json | 8 + .../mitchellh/goamz/testutil/http.go | 180 ++++ .../mitchellh/goamz/testutil/suite.go | 30 + .../src/github.com/motain/gocheck/.bzrignore | 3 + .../src/github.com/motain/gocheck/.gitignore | 4 + .../src/github.com/motain/gocheck/LICENSE | 29 + .../src/github.com/motain/gocheck/Makefile | 30 + .../src/github.com/motain/gocheck/TODO | 2 + .../github.com/motain/gocheck/benchmark.go | 136 +++ .../motain/gocheck/benchmark_test.go | 75 ++ .../motain/gocheck/bootstrap_test.go | 82 ++ .../src/github.com/motain/gocheck/checkers.go | 458 +++++++++ .../motain/gocheck/checkers_test.go | 272 ++++++ .../github.com/motain/gocheck/export_test.go | 9 + .../github.com/motain/gocheck/fixture_test.go | 479 +++++++++ .../motain/gocheck/foundation_test.go | 340 +++++++ .../src/github.com/motain/gocheck/gocheck.go | 916 ++++++++++++++++++ .../github.com/motain/gocheck/gocheck_test.go | 196 ++++ .../src/github.com/motain/gocheck/helpers.go | 221 +++++ .../github.com/motain/gocheck/helpers_test.go | 491 ++++++++++ .../src/github.com/motain/gocheck/printer.go | 168 ++++ .../github.com/motain/gocheck/printer_test.go | 104 ++ .../src/github.com/motain/gocheck/run.go | 152 +++ .../src/github.com/motain/gocheck/run_test.go | 398 ++++++++ 24 files changed, 4783 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/http.go create mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/suite.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/.bzrignore create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/.gitignore create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/LICENSE create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/Makefile create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/TODO create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/benchmark.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/benchmark_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/bootstrap_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/checkers.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/checkers_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/export_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/fixture_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/foundation_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/gocheck.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/gocheck_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/helpers.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/helpers_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/printer.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/printer_test.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/run.go create mode 100644 Godeps/_workspace/src/github.com/motain/gocheck/run_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 761916140..aeeb0b3bb 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -26,6 +26,14 @@ "ImportPath": "github.com/mitchellh/goamz/s3", "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" }, + { + "ImportPath": "github.com/mitchellh/goamz/testutil", + "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" + }, + { + "ImportPath": "github.com/motain/gocheck", + "Rev": "9beb271d26e640863a5bf4a3c5ea40ccdd466b84" + }, { "ImportPath": "github.com/pkg/sftp", "Rev": "506297c9013d2893d5c5daaa9155e7333a1c58de" diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/http.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/http.go new file mode 100644 index 000000000..ccc570cdd --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/http.go @@ -0,0 +1,180 @@ +package testutil + +import ( + "bytes" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "time" +) + +type HTTPServer struct { + URL string + Timeout time.Duration + started bool + request chan *http.Request + response chan ResponseFunc +} + +type Response struct { + Status int + Headers map[string]string + Body string +} + +var DefaultClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, +} + +func NewHTTPServer() *HTTPServer { + return &HTTPServer{URL: "http://localhost:4444", Timeout: 5 * time.Second} +} + +type ResponseFunc func(path string) Response + +func (s *HTTPServer) Start() { + if s.started { + return + } + s.started = true + s.request = make(chan *http.Request, 1024) + s.response = make(chan ResponseFunc, 1024) + u, err := url.Parse(s.URL) + if err != nil { + panic(err) + } + l, err := net.Listen("tcp", u.Host) + if err != nil { + panic(err) + } + go http.Serve(l, s) + + s.Response(203, nil, "") + for { + // Wait for it to be up. + resp, err := http.Get(s.URL) + if err == nil && resp.StatusCode == 203 { + break + } + time.Sleep(1e8) + } + s.WaitRequest() // Consume dummy request. +} + +// Flush discards all pending requests and responses. +func (s *HTTPServer) Flush() { + for { + select { + case <-s.request: + case <-s.response: + default: + return + } + } +} + +func body(req *http.Request) string { + data, err := ioutil.ReadAll(req.Body) + if err != nil { + panic(err) + } + return string(data) +} + +func (s *HTTPServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { + req.ParseMultipartForm(1e6) + data, err := ioutil.ReadAll(req.Body) + if err != nil { + panic(err) + } + req.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + s.request <- req + var resp Response + select { + case respFunc := <-s.response: + resp = respFunc(req.URL.Path) + case <-time.After(s.Timeout): + const msg = "ERROR: Timeout waiting for test to prepare a response\n" + fmt.Fprintf(os.Stderr, msg) + resp = Response{500, nil, msg} + } + if resp.Headers != nil { + h := w.Header() + for k, v := range resp.Headers { + h.Set(k, v) + } + } + if resp.Status != 0 { + w.WriteHeader(resp.Status) + } + w.Write([]byte(resp.Body)) +} + +// WaitRequests returns the next n requests made to the http server from +// the queue. If not enough requests were previously made, it waits until +// the timeout value for them to be made. +func (s *HTTPServer) WaitRequests(n int) []*http.Request { + reqs := make([]*http.Request, 0, n) + for i := 0; i < n; i++ { + select { + case req := <-s.request: + reqs = append(reqs, req) + case <-time.After(s.Timeout): + panic("Timeout waiting for request") + } + } + return reqs +} + +// WaitRequest returns the next request made to the http server from +// the queue. If no requests were previously made, it waits until the +// timeout value for one to be made. +func (s *HTTPServer) WaitRequest() *http.Request { + return s.WaitRequests(1)[0] +} + +// ResponseFunc prepares the test server to respond the following n +// requests using f to build each response. +func (s *HTTPServer) ResponseFunc(n int, f ResponseFunc) { + for i := 0; i < n; i++ { + s.response <- f + } +} + +// ResponseMap maps request paths to responses. +type ResponseMap map[string]Response + +// ResponseMap prepares the test server to respond the following n +// requests using the m to obtain the responses. +func (s *HTTPServer) ResponseMap(n int, m ResponseMap) { + f := func(path string) Response { + for rpath, resp := range m { + if rpath == path { + return resp + } + } + body := "Path not found in response map: " + path + return Response{Status: 500, Body: body} + } + s.ResponseFunc(n, f) +} + +// Responses prepares the test server to respond the following n requests +// using the provided response parameters. +func (s *HTTPServer) Responses(n int, status int, headers map[string]string, body string) { + f := func(path string) Response { + return Response{status, headers, body} + } + s.ResponseFunc(n, f) +} + +// Response prepares the test server to respond the following request +// using the provided response parameters. +func (s *HTTPServer) Response(status int, headers map[string]string, body string) { + s.Responses(1, status, headers, body) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/suite.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/suite.go new file mode 100644 index 000000000..2d432a00d --- /dev/null +++ b/Godeps/_workspace/src/github.com/mitchellh/goamz/testutil/suite.go @@ -0,0 +1,30 @@ +package testutil + +import ( + "flag" + "github.com/mitchellh/goamz/aws" + . "github.com/motain/gocheck" +) + +// Amazon must be used by all tested packages to determine whether to +// run functional tests against the real AWS servers. +var Amazon bool + +func init() { + flag.BoolVar(&Amazon, "amazon", false, "Enable tests against amazon server") +} + +type LiveSuite struct { + auth aws.Auth +} + +func (s *LiveSuite) SetUpSuite(c *C) { + if !Amazon { + c.Skip("amazon tests not enabled (-amazon flag)") + } + auth, err := aws.EnvAuth() + if err != nil { + c.Fatal(err.Error()) + } + s.auth = auth +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/.bzrignore b/Godeps/_workspace/src/github.com/motain/gocheck/.bzrignore new file mode 100644 index 000000000..e39772eb9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/.bzrignore @@ -0,0 +1,3 @@ +_* +[856].out +[856].out.exe diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/.gitignore b/Godeps/_workspace/src/github.com/motain/gocheck/.gitignore new file mode 100644 index 000000000..191a5360b --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/.gitignore @@ -0,0 +1,4 @@ +_* +*.swp +*.[568] +[568].out diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/LICENSE b/Godeps/_workspace/src/github.com/motain/gocheck/LICENSE new file mode 100644 index 000000000..d106fac0e --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/LICENSE @@ -0,0 +1,29 @@ +Gocheck - A rich testing framework for Go + +Copyright (c) 2010, Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/Makefile b/Godeps/_workspace/src/github.com/motain/gocheck/Makefile new file mode 100644 index 000000000..333e56cc1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/Makefile @@ -0,0 +1,30 @@ +include $(GOROOT)/src/Make.inc + +TARG=launchpad.net/gocheck + +GOFILES=\ + gocheck.go\ + helpers.go\ + run.go\ + checkers.go\ + printer.go\ + +#TARGDIR=$(GOPATH)/pkg/$(GOOS)_$(GOARCH) +#GCIMPORTS=$(patsubst %,-I %/pkg/$(GOOS)_$(GOARCH),$(subst :, ,$(GOPATH))) +#LDIMPORTS=$(patsubst %,-L %/pkg/$(GOOS)_$(GOARCH),$(subst :, ,$(GOPATH))) + +include $(GOROOT)/src/Make.pkg + +GOFMT=gofmt + +BADFMT=$(shell $(GOFMT) -l $(GOFILES) $(filter-out printer_test.go,$(wildcard *_test.go))) + +gofmt: $(BADFMT) + @for F in $(BADFMT); do $(GOFMT) -w $$F && echo $$F; done + +ifneq ($(BADFMT),) +ifneq ($(MAKECMDGOALS),gofmt) +#$(warning WARNING: make gofmt: $(BADFMT)) +endif +endif + diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/TODO b/Godeps/_workspace/src/github.com/motain/gocheck/TODO new file mode 100644 index 000000000..33498270e --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/TODO @@ -0,0 +1,2 @@ +- Assert(slice, Contains, item) +- Parallel test support diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/benchmark.go b/Godeps/_workspace/src/github.com/motain/gocheck/benchmark.go new file mode 100644 index 000000000..dcd426f22 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/benchmark.go @@ -0,0 +1,136 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gocheck + +import ( + "fmt" + "time" +) + +// testingB is a type passed to Benchmark functions to manage benchmark +// timing and to specify the number of iterations to run. +type timer struct { + start time.Time // Time test or benchmark started + duration time.Duration + N int + bytes int64 + timerOn bool + benchTime time.Duration +} + +// StartTimer starts timing a test. This function is called automatically +// before a benchmark starts, but it can also used to resume timing after +// a call to StopTimer. +func (c *C) StartTimer() { + if !c.timerOn { + c.start = time.Now() + c.timerOn = true + } +} + +// StopTimer stops timing a test. This can be used to pause the timer +// while performing complex initialization that you don't +// want to measure. +func (c *C) StopTimer() { + if c.timerOn { + c.duration += time.Now().Sub(c.start) + c.timerOn = false + } +} + +// ResetTimer sets the elapsed benchmark time to zero. +// It does not affect whether the timer is running. +func (c *C) ResetTimer() { + if c.timerOn { + c.start = time.Now() + } + c.duration = 0 +} + +// SetBytes informs the number of bytes that the benchmark processes +// on each iteration. If this is called in a benchmark it will also +// report MB/s. +func (c *C) SetBytes(n int64) { + c.bytes = n +} + +func (c *C) nsPerOp() int64 { + if c.N <= 0 { + return 0 + } + return c.duration.Nanoseconds() / int64(c.N) +} + +func (c *C) mbPerSec() float64 { + if c.bytes <= 0 || c.duration <= 0 || c.N <= 0 { + return 0 + } + return (float64(c.bytes) * float64(c.N) / 1e6) / c.duration.Seconds() +} + +func (c *C) timerString() string { + if c.N <= 0 { + return fmt.Sprintf("%3.3fs", float64(c.duration.Nanoseconds())/1e9) + } + mbs := c.mbPerSec() + mb := "" + if mbs != 0 { + mb = fmt.Sprintf("\t%7.2f MB/s", mbs) + } + nsop := c.nsPerOp() + ns := fmt.Sprintf("%10d ns/op", nsop) + if c.N > 0 && nsop < 100 { + // The format specifiers here make sure that + // the ones digits line up for all three possible formats. + if nsop < 10 { + ns = fmt.Sprintf("%13.2f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) + } else { + ns = fmt.Sprintf("%12.1f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) + } + } + return fmt.Sprintf("%8d\t%s%s", c.N, ns, mb) +} + +func min(x, y int) int { + if x > y { + return y + } + return x +} + +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// roundDown10 rounds a number down to the nearest power of 10. +func roundDown10(n int) int { + var tens = 0 + // tens = floor(log_10(n)) + for n > 10 { + n = n / 10 + tens++ + } + // result = 10^tens + result := 1 + for i := 0; i < tens; i++ { + result *= 10 + } + return result +} + +// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. +func roundUp(n int) int { + base := roundDown10(n) + if n < (2 * base) { + return 2 * base + } + if n < (5 * base) { + return 5 * base + } + return 10 * base +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/benchmark_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/benchmark_test.go new file mode 100644 index 000000000..7e96ae7ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/benchmark_test.go @@ -0,0 +1,75 @@ +// These tests verify the test running logic. + +package gocheck_test + +import ( + . "launchpad.net/gocheck" + "time" +) + +var benchmarkS = Suite(&BenchmarkS{}) + +type BenchmarkS struct{} + +func (s *BenchmarkS) TestCountSuite(c *C) { + suitesRun += 1 +} + +func (s *BenchmarkS) TestBasicTestTiming(c *C) { + helper := FixtureHelper{sleepOn: "Test1", sleep: 1000000 * time.Nanosecond} + output := String{} + runConf := RunConf{Output: &output, Verbose: true} + Run(&helper, &runConf) + + expected := "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Test1\t0\\.001s\n" + + "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Test2\t0\\.000s\n" + c.Assert(output.value, Matches, expected) +} + +func (s *BenchmarkS) TestStreamTestTiming(c *C) { + helper := FixtureHelper{sleepOn: "SetUpSuite", sleep: 1000000 * time.Nanosecond} + output := String{} + runConf := RunConf{Output: &output, Stream: true} + Run(&helper, &runConf) + + expected := "(?s).*\nPASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.SetUpSuite\t *0\\.001s\n.*" + c.Assert(output.value, Matches, expected) +} + +func (s *BenchmarkS) TestBenchmark(c *C) { + helper := FixtureHelper{sleep: 100000} + output := String{} + runConf := RunConf{ + Output: &output, + Benchmark: true, + BenchmarkTime: 10000000, + Filter: "Benchmark1", + } + Run(&helper, &runConf) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Benchmark1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Benchmark1") + c.Check(helper.calls[6], Equals, "TearDownTest") + // ... and more. + + expected := "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Benchmark1\t *100\t *[12][0-9]{5} ns/op\n" + c.Assert(output.value, Matches, expected) +} + +func (s *BenchmarkS) TestBenchmarkBytes(c *C) { + helper := FixtureHelper{sleep: 100000} + output := String{} + runConf := RunConf{ + Output: &output, + Benchmark: true, + BenchmarkTime: 10000000, + Filter: "Benchmark2", + } + Run(&helper, &runConf) + + expected := "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Benchmark2\t *100\t *[12][0-9]{5} ns/op\t *[4-9]\\.[0-9]{2} MB/s\n" + c.Assert(output.value, Matches, expected) +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/bootstrap_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/bootstrap_test.go new file mode 100644 index 000000000..e50d9ea1e --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/bootstrap_test.go @@ -0,0 +1,82 @@ +// These initial tests are for bootstrapping. They verify that we can +// basically use the testing infrastructure itself to check if the test +// system is working. +// +// These tests use will break down the test runner badly in case of +// errors because if they simply fail, we can't be sure the developer +// will ever see anything (because failing means the failing system +// somehow isn't working! :-) +// +// Do not assume *any* internal functionality works as expected besides +// what's actually tested here. + +package gocheck_test + +import ( + "fmt" + "launchpad.net/gocheck" + "strings" +) + +type BootstrapS struct{} + +var boostrapS = gocheck.Suite(&BootstrapS{}) + +func (s *BootstrapS) TestCountSuite(c *gocheck.C) { + suitesRun += 1 +} + +func (s *BootstrapS) TestFailedAndFail(c *gocheck.C) { + if c.Failed() { + critical("c.Failed() must be false first!") + } + c.Fail() + if !c.Failed() { + critical("c.Fail() didn't put the test in a failed state!") + } + c.Succeed() +} + +func (s *BootstrapS) TestFailedAndSucceed(c *gocheck.C) { + c.Fail() + c.Succeed() + if c.Failed() { + critical("c.Succeed() didn't put the test back in a non-failed state") + } +} + +func (s *BootstrapS) TestLogAndGetTestLog(c *gocheck.C) { + c.Log("Hello there!") + log := c.GetTestLog() + if log != "Hello there!\n" { + critical(fmt.Sprintf("Log() or GetTestLog() is not working! Got: %#v", log)) + } +} + +func (s *BootstrapS) TestLogfAndGetTestLog(c *gocheck.C) { + c.Logf("Hello %v", "there!") + log := c.GetTestLog() + if log != "Hello there!\n" { + critical(fmt.Sprintf("Logf() or GetTestLog() is not working! Got: %#v", log)) + } +} + +func (s *BootstrapS) TestRunShowsErrors(c *gocheck.C) { + output := String{} + gocheck.Run(&FailHelper{}, &gocheck.RunConf{Output: &output}) + if strings.Index(output.value, "Expected failure!") == -1 { + critical(fmt.Sprintf("RunWithWriter() output did not contain the "+ + "expected failure! Got: %#v", + output.value)) + } +} + +func (s *BootstrapS) TestRunDoesntShowSuccesses(c *gocheck.C) { + output := String{} + gocheck.Run(&SuccessHelper{}, &gocheck.RunConf{Output: &output}) + if strings.Index(output.value, "Expected success!") != -1 { + critical(fmt.Sprintf("RunWithWriter() output contained a successful "+ + "test! Got: %#v", + output.value)) + } +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/checkers.go b/Godeps/_workspace/src/github.com/motain/gocheck/checkers.go new file mode 100644 index 000000000..5c91200f0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/checkers.go @@ -0,0 +1,458 @@ +package gocheck + +import ( + "fmt" + "reflect" + "regexp" +) + +// ----------------------------------------------------------------------- +// CommentInterface and Commentf helper, to attach extra information to checks. + +type comment struct { + format string + args []interface{} +} + +// Commentf returns an infomational value to use with Assert or Check calls. +// If the checker test fails, the provided arguments will be passed to +// fmt.Sprintf, and will be presented next to the logged failure. +// +// For example: +// +// c.Assert(v, Equals, 42, Commentf("Iteration #%d failed.", i)) +// +// Note that if the comment is constant, a better option is to +// simply use a normal comment right above or next to the line, as +// it will also get printed with any errors: +// +// c.Assert(l, Equals, 8192) // Ensure buffer size is correct (bug #123) +// +func Commentf(format string, args ...interface{}) CommentInterface { + return &comment{format, args} +} + +// CommentInterface must be implemented by types that attach extra +// information to failed checks. See the Commentf function for details. +type CommentInterface interface { + CheckCommentString() string +} + +func (c *comment) CheckCommentString() string { + return fmt.Sprintf(c.format, c.args...) +} + +// ----------------------------------------------------------------------- +// The Checker interface. + +// The Checker interface must be provided by checkers used with +// the Assert and Check verification methods. +type Checker interface { + Info() *CheckerInfo + Check(params []interface{}, names []string) (result bool, error string) +} + +// See the Checker interface. +type CheckerInfo struct { + Name string + Params []string +} + +func (info *CheckerInfo) Info() *CheckerInfo { + return info +} + +// ----------------------------------------------------------------------- +// Not checker logic inverter. + +// The Not checker inverts the logic of the provided checker. The +// resulting checker will succeed where the original one failed, and +// vice-versa. +// +// For example: +// +// c.Assert(a, Not(Equals), b) +// +func Not(checker Checker) Checker { + return ¬Checker{checker} +} + +type notChecker struct { + sub Checker +} + +func (checker *notChecker) Info() *CheckerInfo { + info := *checker.sub.Info() + info.Name = "Not(" + info.Name + ")" + return &info +} + +func (checker *notChecker) Check(params []interface{}, names []string) (result bool, error string) { + result, error = checker.sub.Check(params, names) + result = !result + return +} + +// ----------------------------------------------------------------------- +// IsNil checker. + +type isNilChecker struct { + *CheckerInfo +} + +// The IsNil checker tests whether the obtained value is nil. +// +// For example: +// +// c.Assert(err, IsNil) +// +var IsNil Checker = &isNilChecker{ + &CheckerInfo{Name: "IsNil", Params: []string{"value"}}, +} + +func (checker *isNilChecker) Check(params []interface{}, names []string) (result bool, error string) { + return isNil(params[0]), "" +} + +func isNil(obtained interface{}) (result bool) { + if obtained == nil { + result = true + } else { + switch v := reflect.ValueOf(obtained); v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + } + } + return +} + +// ----------------------------------------------------------------------- +// NotNil checker. Alias for Not(IsNil), since it's so common. + +type notNilChecker struct { + *CheckerInfo +} + +// The NotNil checker verifies that the obtained value is not nil. +// +// For example: +// +// c.Assert(iface, NotNil) +// +// This is an alias for Not(IsNil), made available since it's a +// fairly common check. +// +var NotNil Checker = ¬NilChecker{ + &CheckerInfo{Name: "NotNil", Params: []string{"value"}}, +} + +func (checker *notNilChecker) Check(params []interface{}, names []string) (result bool, error string) { + return !isNil(params[0]), "" +} + +// ----------------------------------------------------------------------- +// Equals checker. + +type equalsChecker struct { + *CheckerInfo +} + +// The Equals checker verifies that the obtained value is equal to +// the expected value, according to usual Go semantics for ==. +// +// For example: +// +// c.Assert(value, Equals, 42) +// +var Equals Checker = &equalsChecker{ + &CheckerInfo{Name: "Equals", Params: []string{"obtained", "expected"}}, +} + +func (checker *equalsChecker) Check(params []interface{}, names []string) (result bool, error string) { + defer func() { + if v := recover(); v != nil { + result = false + error = fmt.Sprint(v) + } + }() + return params[0] == params[1], "" +} + +// ----------------------------------------------------------------------- +// DeepEquals checker. + +type deepEqualsChecker struct { + *CheckerInfo +} + +// The DeepEquals checker verifies that the obtained value is deep-equal to +// the expected value. The check will work correctly even when facing +// slices, interfaces, and values of different types (which always fail +// the test). +// +// For example: +// +// c.Assert(value, DeepEquals, 42) +// c.Assert(array, DeepEquals, []string{"hi", "there"}) +// +var DeepEquals Checker = &deepEqualsChecker{ + &CheckerInfo{Name: "DeepEquals", Params: []string{"obtained", "expected"}}, +} + +func (checker *deepEqualsChecker) Check(params []interface{}, names []string) (result bool, error string) { + return reflect.DeepEqual(params[0], params[1]), "" +} + +// ----------------------------------------------------------------------- +// HasLen checker. + +type hasLenChecker struct { + *CheckerInfo +} + +// The HasLen checker verifies that the obtained value has the +// provided length. In many cases this is superior to using Equals +// in conjuction with the len function because in case the check +// fails the value itself will be printed, instead of its length, +// providing more details for figuring the problem. +// +// For example: +// +// c.Assert(list, HasLen, 5) +// +var HasLen Checker = &hasLenChecker{ + &CheckerInfo{Name: "HasLen", Params: []string{"obtained", "n"}}, +} + +func (checker *hasLenChecker) Check(params []interface{}, names []string) (result bool, error string) { + n, ok := params[1].(int) + if !ok { + return false, "n must be an int" + } + value := reflect.ValueOf(params[0]) + switch value.Kind() { + case reflect.Map, reflect.Array, reflect.Slice, reflect.Chan, reflect.String: + default: + return false, "obtained value type has no length" + } + return value.Len() == n, "" +} + +// ----------------------------------------------------------------------- +// ErrorMatches checker. + +type errorMatchesChecker struct { + *CheckerInfo +} + +// The ErrorMatches checker verifies that the error value +// is non nil and matches the regular expression provided. +// +// For example: +// +// c.Assert(err, ErrorMatches, "perm.*denied") +// +var ErrorMatches Checker = errorMatchesChecker{ + &CheckerInfo{Name: "ErrorMatches", Params: []string{"value", "regex"}}, +} + +func (checker errorMatchesChecker) Check(params []interface{}, names []string) (result bool, errStr string) { + if params[0] == nil { + return false, "Error value is nil" + } + err, ok := params[0].(error) + if !ok { + return false, "Value is not an error" + } + params[0] = err.Error() + names[0] = "error" + return matches(params[0], params[1]) +} + +// ----------------------------------------------------------------------- +// Matches checker. + +type matchesChecker struct { + *CheckerInfo +} + +// The Matches checker verifies that the string provided as the obtained +// value (or the string resulting from obtained.String()) matches the +// regular expression provided. +// +// For example: +// +// c.Assert(err, Matches, "perm.*denied") +// +var Matches Checker = &matchesChecker{ + &CheckerInfo{Name: "Matches", Params: []string{"value", "regex"}}, +} + +func (checker *matchesChecker) Check(params []interface{}, names []string) (result bool, error string) { + return matches(params[0], params[1]) +} + +func matches(value, regex interface{}) (result bool, error string) { + reStr, ok := regex.(string) + if !ok { + return false, "Regex must be a string" + } + valueStr, valueIsStr := value.(string) + if !valueIsStr { + if valueWithStr, valueHasStr := value.(fmt.Stringer); valueHasStr { + valueStr, valueIsStr = valueWithStr.String(), true + } + } + if valueIsStr { + matches, err := regexp.MatchString("^"+reStr+"$", valueStr) + if err != nil { + return false, "Can't compile regex: " + err.Error() + } + return matches, "" + } + return false, "Obtained value is not a string and has no .String()" +} + +// ----------------------------------------------------------------------- +// Panics checker. + +type panicsChecker struct { + *CheckerInfo +} + +// The Panics checker verifies that calling the provided zero-argument +// function will cause a panic which is deep-equal to the provided value. +// +// For example: +// +// c.Assert(func() { f(1, 2) }, Panics, &SomeErrorType{"BOOM"}). +// +// +var Panics Checker = &panicsChecker{ + &CheckerInfo{Name: "Panics", Params: []string{"function", "expected"}}, +} + +func (checker *panicsChecker) Check(params []interface{}, names []string) (result bool, error string) { + f := reflect.ValueOf(params[0]) + if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { + return false, "Function must take zero arguments" + } + defer func() { + // If the function has not panicked, then don't do the check. + if error != "" { + return + } + params[0] = recover() + names[0] = "panic" + result = reflect.DeepEqual(params[0], params[1]) + }() + f.Call(nil) + return false, "Function has not panicked" +} + +type panicMatchesChecker struct { + *CheckerInfo +} + +// The PanicMatches checker verifies that calling the provided zero-argument +// function will cause a panic with an error value matching +// the regular expression provided. +// +// For example: +// +// c.Assert(func() { f(1, 2) }, PanicMatches, `open.*: no such file or directory`). +// +// +var PanicMatches Checker = &panicMatchesChecker{ + &CheckerInfo{Name: "PanicMatches", Params: []string{"function", "expected"}}, +} + +func (checker *panicMatchesChecker) Check(params []interface{}, names []string) (result bool, errmsg string) { + f := reflect.ValueOf(params[0]) + if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { + return false, "Function must take zero arguments" + } + defer func() { + // If the function has not panicked, then don't do the check. + if errmsg != "" { + return + } + obtained := recover() + names[0] = "panic" + if e, ok := obtained.(error); ok { + params[0] = e.Error() + } else if _, ok := obtained.(string); ok { + params[0] = obtained + } else { + errmsg = "Panic value is not a string or an error" + return + } + result, errmsg = matches(params[0], params[1]) + }() + f.Call(nil) + return false, "Function has not panicked" +} + +// ----------------------------------------------------------------------- +// FitsTypeOf checker. + +type fitsTypeChecker struct { + *CheckerInfo +} + +// The FitsTypeOf checker verifies that the obtained value is +// assignable to a variable with the same type as the provided +// sample value. +// +// For example: +// +// c.Assert(value, FitsTypeOf, int64(0)) +// c.Assert(value, FitsTypeOf, os.Error(nil)) +// +var FitsTypeOf Checker = &fitsTypeChecker{ + &CheckerInfo{Name: "FitsTypeOf", Params: []string{"obtained", "sample"}}, +} + +func (checker *fitsTypeChecker) Check(params []interface{}, names []string) (result bool, error string) { + obtained := reflect.ValueOf(params[0]) + sample := reflect.ValueOf(params[1]) + if !obtained.IsValid() { + return false, "" + } + if !sample.IsValid() { + return false, "Invalid sample value" + } + return obtained.Type().AssignableTo(sample.Type()), "" +} + +// ----------------------------------------------------------------------- +// Implements checker. + +type implementsChecker struct { + *CheckerInfo +} + +// The Implements checker verifies that the obtained value +// implements the interface specified via a pointer to an interface +// variable. +// +// For example: +// +// var e os.Error +// c.Assert(err, Implements, &e) +// +var Implements Checker = &implementsChecker{ + &CheckerInfo{Name: "Implements", Params: []string{"obtained", "ifaceptr"}}, +} + +func (checker *implementsChecker) Check(params []interface{}, names []string) (result bool, error string) { + obtained := reflect.ValueOf(params[0]) + ifaceptr := reflect.ValueOf(params[1]) + if !obtained.IsValid() { + return false, "" + } + if !ifaceptr.IsValid() || ifaceptr.Kind() != reflect.Ptr || ifaceptr.Elem().Kind() != reflect.Interface { + return false, "ifaceptr should be a pointer to an interface variable" + } + return obtained.Type().Implements(ifaceptr.Elem().Type()), "" +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/checkers_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/checkers_test.go new file mode 100644 index 000000000..0945491b9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/checkers_test.go @@ -0,0 +1,272 @@ +package gocheck_test + +import ( + "errors" + "launchpad.net/gocheck" + "reflect" + "runtime" +) + +type CheckersS struct{} + +var _ = gocheck.Suite(&CheckersS{}) + +func testInfo(c *gocheck.C, checker gocheck.Checker, name string, paramNames []string) { + info := checker.Info() + if info.Name != name { + c.Fatalf("Got name %s, expected %s", info.Name, name) + } + if !reflect.DeepEqual(info.Params, paramNames) { + c.Fatalf("Got param names %#v, expected %#v", info.Params, paramNames) + } +} + +func testCheck(c *gocheck.C, checker gocheck.Checker, result bool, error string, params ...interface{}) ([]interface{}, []string) { + info := checker.Info() + if len(params) != len(info.Params) { + c.Fatalf("unexpected param count in test; expected %d got %d", len(info.Params), len(params)) + } + names := append([]string{}, info.Params...) + result_, error_ := checker.Check(params, names) + if result_ != result || error_ != error { + c.Fatalf("%s.Check(%#v) returned (%#v, %#v) rather than (%#v, %#v)", + info.Name, params, result_, error_, result, error) + } + return params, names +} + +func (s *CheckersS) TestComment(c *gocheck.C) { + bug := gocheck.Commentf("a %d bc", 42) + comment := bug.CheckCommentString() + if comment != "a 42 bc" { + c.Fatalf("Commentf returned %#v", comment) + } +} + +func (s *CheckersS) TestIsNil(c *gocheck.C) { + testInfo(c, gocheck.IsNil, "IsNil", []string{"value"}) + + testCheck(c, gocheck.IsNil, true, "", nil) + testCheck(c, gocheck.IsNil, false, "", "a") + + testCheck(c, gocheck.IsNil, true, "", (chan int)(nil)) + testCheck(c, gocheck.IsNil, false, "", make(chan int)) + testCheck(c, gocheck.IsNil, true, "", (error)(nil)) + testCheck(c, gocheck.IsNil, false, "", errors.New("")) + testCheck(c, gocheck.IsNil, true, "", ([]int)(nil)) + testCheck(c, gocheck.IsNil, false, "", make([]int, 1)) + testCheck(c, gocheck.IsNil, false, "", int(0)) +} + +func (s *CheckersS) TestNotNil(c *gocheck.C) { + testInfo(c, gocheck.NotNil, "NotNil", []string{"value"}) + + testCheck(c, gocheck.NotNil, false, "", nil) + testCheck(c, gocheck.NotNil, true, "", "a") + + testCheck(c, gocheck.NotNil, false, "", (chan int)(nil)) + testCheck(c, gocheck.NotNil, true, "", make(chan int)) + testCheck(c, gocheck.NotNil, false, "", (error)(nil)) + testCheck(c, gocheck.NotNil, true, "", errors.New("")) + testCheck(c, gocheck.NotNil, false, "", ([]int)(nil)) + testCheck(c, gocheck.NotNil, true, "", make([]int, 1)) +} + +func (s *CheckersS) TestNot(c *gocheck.C) { + testInfo(c, gocheck.Not(gocheck.IsNil), "Not(IsNil)", []string{"value"}) + + testCheck(c, gocheck.Not(gocheck.IsNil), false, "", nil) + testCheck(c, gocheck.Not(gocheck.IsNil), true, "", "a") +} + +type simpleStruct struct { + i int +} + +func (s *CheckersS) TestEquals(c *gocheck.C) { + testInfo(c, gocheck.Equals, "Equals", []string{"obtained", "expected"}) + + // The simplest. + testCheck(c, gocheck.Equals, true, "", 42, 42) + testCheck(c, gocheck.Equals, false, "", 42, 43) + + // Different native types. + testCheck(c, gocheck.Equals, false, "", int32(42), int64(42)) + + // With nil. + testCheck(c, gocheck.Equals, false, "", 42, nil) + + // Slices + testCheck(c, gocheck.Equals, false, "runtime error: comparing uncomparable type []uint8", []byte{1, 2}, []byte{1, 2}) + + // Struct values + testCheck(c, gocheck.Equals, true, "", simpleStruct{1}, simpleStruct{1}) + testCheck(c, gocheck.Equals, false, "", simpleStruct{1}, simpleStruct{2}) + + // Struct pointers + testCheck(c, gocheck.Equals, false, "", &simpleStruct{1}, &simpleStruct{1}) + testCheck(c, gocheck.Equals, false, "", &simpleStruct{1}, &simpleStruct{2}) +} + +func (s *CheckersS) TestDeepEquals(c *gocheck.C) { + testInfo(c, gocheck.DeepEquals, "DeepEquals", []string{"obtained", "expected"}) + + // The simplest. + testCheck(c, gocheck.DeepEquals, true, "", 42, 42) + testCheck(c, gocheck.DeepEquals, false, "", 42, 43) + + // Different native types. + testCheck(c, gocheck.DeepEquals, false, "", int32(42), int64(42)) + + // With nil. + testCheck(c, gocheck.DeepEquals, false, "", 42, nil) + + // Slices + testCheck(c, gocheck.DeepEquals, true, "", []byte{1, 2}, []byte{1, 2}) + testCheck(c, gocheck.DeepEquals, false, "", []byte{1, 2}, []byte{1, 3}) + + // Struct values + testCheck(c, gocheck.DeepEquals, true, "", simpleStruct{1}, simpleStruct{1}) + testCheck(c, gocheck.DeepEquals, false, "", simpleStruct{1}, simpleStruct{2}) + + // Struct pointers + testCheck(c, gocheck.DeepEquals, true, "", &simpleStruct{1}, &simpleStruct{1}) + testCheck(c, gocheck.DeepEquals, false, "", &simpleStruct{1}, &simpleStruct{2}) +} + +func (s *CheckersS) TestHasLen(c *gocheck.C) { + testInfo(c, gocheck.HasLen, "HasLen", []string{"obtained", "n"}) + + testCheck(c, gocheck.HasLen, true, "", "abcd", 4) + testCheck(c, gocheck.HasLen, true, "", []int{1, 2}, 2) + testCheck(c, gocheck.HasLen, false, "", []int{1, 2}, 3) + + testCheck(c, gocheck.HasLen, false, "n must be an int", []int{1, 2}, "2") + testCheck(c, gocheck.HasLen, false, "obtained value type has no length", nil, 2) +} + +func (s *CheckersS) TestErrorMatches(c *gocheck.C) { + testInfo(c, gocheck.ErrorMatches, "ErrorMatches", []string{"value", "regex"}) + + testCheck(c, gocheck.ErrorMatches, false, "Error value is nil", nil, "some error") + testCheck(c, gocheck.ErrorMatches, false, "Value is not an error", 1, "some error") + testCheck(c, gocheck.ErrorMatches, true, "", errors.New("some error"), "some error") + testCheck(c, gocheck.ErrorMatches, true, "", errors.New("some error"), "so.*or") + + // Verify params mutation + params, names := testCheck(c, gocheck.ErrorMatches, false, "", errors.New("some error"), "other error") + c.Assert(params[0], gocheck.Equals, "some error") + c.Assert(names[0], gocheck.Equals, "error") +} + +func (s *CheckersS) TestMatches(c *gocheck.C) { + testInfo(c, gocheck.Matches, "Matches", []string{"value", "regex"}) + + // Simple matching + testCheck(c, gocheck.Matches, true, "", "abc", "abc") + testCheck(c, gocheck.Matches, true, "", "abc", "a.c") + + // Must match fully + testCheck(c, gocheck.Matches, false, "", "abc", "ab") + testCheck(c, gocheck.Matches, false, "", "abc", "bc") + + // String()-enabled values accepted + testCheck(c, gocheck.Matches, true, "", reflect.ValueOf("abc"), "a.c") + testCheck(c, gocheck.Matches, false, "", reflect.ValueOf("abc"), "a.d") + + // Some error conditions. + testCheck(c, gocheck.Matches, false, "Obtained value is not a string and has no .String()", 1, "a.c") + testCheck(c, gocheck.Matches, false, "Can't compile regex: error parsing regexp: missing closing ]: `[c$`", "abc", "a[c") +} + +func (s *CheckersS) TestPanics(c *gocheck.C) { + testInfo(c, gocheck.Panics, "Panics", []string{"function", "expected"}) + + // Some errors. + testCheck(c, gocheck.Panics, false, "Function has not panicked", func() bool { return false }, "BOOM") + testCheck(c, gocheck.Panics, false, "Function must take zero arguments", 1, "BOOM") + + // Plain strings. + testCheck(c, gocheck.Panics, true, "", func() { panic("BOOM") }, "BOOM") + testCheck(c, gocheck.Panics, false, "", func() { panic("KABOOM") }, "BOOM") + testCheck(c, gocheck.Panics, true, "", func() bool { panic("BOOM") }, "BOOM") + + // Error values. + testCheck(c, gocheck.Panics, true, "", func() { panic(errors.New("BOOM")) }, errors.New("BOOM")) + testCheck(c, gocheck.Panics, false, "", func() { panic(errors.New("KABOOM")) }, errors.New("BOOM")) + + type deep struct{ i int } + // Deep value + testCheck(c, gocheck.Panics, true, "", func() { panic(&deep{99}) }, &deep{99}) + + // Verify params/names mutation + params, names := testCheck(c, gocheck.Panics, false, "", func() { panic(errors.New("KABOOM")) }, errors.New("BOOM")) + c.Assert(params[0], gocheck.ErrorMatches, "KABOOM") + c.Assert(names[0], gocheck.Equals, "panic") + + // Verify a nil panic + testCheck(c, gocheck.Panics, true, "", func() { panic(nil) }, nil) + testCheck(c, gocheck.Panics, false, "", func() { panic(nil) }, "NOPE") +} + +func (s *CheckersS) TestPanicMatches(c *gocheck.C) { + testInfo(c, gocheck.PanicMatches, "PanicMatches", []string{"function", "expected"}) + + // Error matching. + testCheck(c, gocheck.PanicMatches, true, "", func() { panic(errors.New("BOOM")) }, "BO.M") + testCheck(c, gocheck.PanicMatches, false, "", func() { panic(errors.New("KABOOM")) }, "BO.M") + + // Some errors. + testCheck(c, gocheck.PanicMatches, false, "Function has not panicked", func() bool { return false }, "BOOM") + testCheck(c, gocheck.PanicMatches, false, "Function must take zero arguments", 1, "BOOM") + + // Plain strings. + testCheck(c, gocheck.PanicMatches, true, "", func() { panic("BOOM") }, "BO.M") + testCheck(c, gocheck.PanicMatches, false, "", func() { panic("KABOOM") }, "BOOM") + testCheck(c, gocheck.PanicMatches, true, "", func() bool { panic("BOOM") }, "BO.M") + + // Verify params/names mutation + params, names := testCheck(c, gocheck.PanicMatches, false, "", func() { panic(errors.New("KABOOM")) }, "BOOM") + c.Assert(params[0], gocheck.Equals, "KABOOM") + c.Assert(names[0], gocheck.Equals, "panic") + + // Verify a nil panic + testCheck(c, gocheck.PanicMatches, false, "Panic value is not a string or an error", func() { panic(nil) }, "") +} + +func (s *CheckersS) TestFitsTypeOf(c *gocheck.C) { + testInfo(c, gocheck.FitsTypeOf, "FitsTypeOf", []string{"obtained", "sample"}) + + // Basic types + testCheck(c, gocheck.FitsTypeOf, true, "", 1, 0) + testCheck(c, gocheck.FitsTypeOf, false, "", 1, int64(0)) + + // Aliases + testCheck(c, gocheck.FitsTypeOf, false, "", 1, errors.New("")) + testCheck(c, gocheck.FitsTypeOf, false, "", "error", errors.New("")) + testCheck(c, gocheck.FitsTypeOf, true, "", errors.New("error"), errors.New("")) + + // Structures + testCheck(c, gocheck.FitsTypeOf, false, "", 1, simpleStruct{}) + testCheck(c, gocheck.FitsTypeOf, false, "", simpleStruct{42}, &simpleStruct{}) + testCheck(c, gocheck.FitsTypeOf, true, "", simpleStruct{42}, simpleStruct{}) + testCheck(c, gocheck.FitsTypeOf, true, "", &simpleStruct{42}, &simpleStruct{}) + + // Some bad values + testCheck(c, gocheck.FitsTypeOf, false, "Invalid sample value", 1, interface{}(nil)) + testCheck(c, gocheck.FitsTypeOf, false, "", interface{}(nil), 0) +} + +func (s *CheckersS) TestImplements(c *gocheck.C) { + testInfo(c, gocheck.Implements, "Implements", []string{"obtained", "ifaceptr"}) + + var e error + var re runtime.Error + testCheck(c, gocheck.Implements, true, "", errors.New(""), &e) + testCheck(c, gocheck.Implements, false, "", errors.New(""), &re) + + // Some bad values + testCheck(c, gocheck.Implements, false, "ifaceptr should be a pointer to an interface variable", 0, errors.New("")) + testCheck(c, gocheck.Implements, false, "ifaceptr should be a pointer to an interface variable", 0, interface{}(nil)) + testCheck(c, gocheck.Implements, false, "", interface{}(nil), &e) +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/export_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/export_test.go new file mode 100644 index 000000000..ad2962199 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/export_test.go @@ -0,0 +1,9 @@ +package gocheck + +func PrintLine(filename string, line int) (string, error) { + return printLine(filename, line) +} + +func Indent(s, with string) string { + return indent(s, with) +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/fixture_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/fixture_test.go new file mode 100644 index 000000000..63f392e49 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/fixture_test.go @@ -0,0 +1,479 @@ +// Tests for the behavior of the test fixture system. + +package gocheck_test + +import ( + . "launchpad.net/gocheck" +) + +// ----------------------------------------------------------------------- +// Fixture test suite. + +type FixtureS struct{} + +var fixtureS = Suite(&FixtureS{}) + +func (s *FixtureS) TestCountSuite(c *C) { + suitesRun += 1 +} + +// ----------------------------------------------------------------------- +// Basic fixture ordering verification. + +func (s *FixtureS) TestOrder(c *C) { + helper := FixtureHelper{} + Run(&helper, nil) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Test2") + c.Check(helper.calls[6], Equals, "TearDownTest") + c.Check(helper.calls[7], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 8) +} + +// ----------------------------------------------------------------------- +// Check the behavior when panics occur within tests and fixtures. + +func (s *FixtureS) TestPanicOnTest(c *C) { + helper := FixtureHelper{panicOn: "Test1"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Test2") + c.Check(helper.calls[6], Equals, "TearDownTest") + c.Check(helper.calls[7], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 8) + + expected := "^\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: FixtureHelper.Test1\n\n" + + "\\.\\.\\. Panic: Test1 \\(PC=[xA-F0-9]+\\)\n\n" + + ".+:[0-9]+\n" + + " in panic\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.trace\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.Test1\n$" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnSetUpTest(c *C) { + helper := FixtureHelper{panicOn: "SetUpTest"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "TearDownTest") + c.Check(helper.calls[3], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 4) + + expected := "^\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper\\.SetUpTest\n\n" + + "\\.\\.\\. Panic: SetUpTest \\(PC=[xA-F0-9]+\\)\n\n" + + ".+:[0-9]+\n" + + " in panic\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.trace\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.SetUpTest\n" + + "\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper\\.Test1\n\n" + + "\\.\\.\\. Panic: Fixture has panicked " + + "\\(see related PANIC\\)\n$" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnTearDownTest(c *C) { + helper := FixtureHelper{panicOn: "TearDownTest"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 5) + + expected := "^\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper.TearDownTest\n\n" + + "\\.\\.\\. Panic: TearDownTest \\(PC=[xA-F0-9]+\\)\n\n" + + ".+:[0-9]+\n" + + " in panic\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.trace\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.TearDownTest\n" + + "\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper\\.Test1\n\n" + + "\\.\\.\\. Panic: Fixture has panicked " + + "\\(see related PANIC\\)\n$" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnSetUpSuite(c *C) { + helper := FixtureHelper{panicOn: "SetUpSuite"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 2) + + expected := "^\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper.SetUpSuite\n\n" + + "\\.\\.\\. Panic: SetUpSuite \\(PC=[xA-F0-9]+\\)\n\n" + + ".+:[0-9]+\n" + + " in panic\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.trace\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.SetUpSuite\n$" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnTearDownSuite(c *C) { + helper := FixtureHelper{panicOn: "TearDownSuite"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Test2") + c.Check(helper.calls[6], Equals, "TearDownTest") + c.Check(helper.calls[7], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 8) + + expected := "^\n-+\n" + + "PANIC: gocheck_test\\.go:[0-9]+: " + + "FixtureHelper.TearDownSuite\n\n" + + "\\.\\.\\. Panic: TearDownSuite \\(PC=[xA-F0-9]+\\)\n\n" + + ".+:[0-9]+\n" + + " in panic\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.trace\n" + + ".*gocheck_test.go:[0-9]+\n" + + " in FixtureHelper.TearDownSuite\n$" + + c.Check(output.value, Matches, expected) +} + +// ----------------------------------------------------------------------- +// A wrong argument on a test or fixture will produce a nice error. + +func (s *FixtureS) TestPanicOnWrongTestArg(c *C) { + helper := WrongTestArgHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "TearDownTest") + c.Check(helper.calls[3], Equals, "SetUpTest") + c.Check(helper.calls[4], Equals, "Test2") + c.Check(helper.calls[5], Equals, "TearDownTest") + c.Check(helper.calls[6], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 7) + + expected := "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongTestArgHelper\\.Test1\n\n" + + "\\.\\.\\. Panic: WrongTestArgHelper\\.Test1 argument " + + "should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnWrongSetUpTestArg(c *C) { + helper := WrongSetUpTestArgHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(len(helper.calls), Equals, 0) + + expected := + "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongSetUpTestArgHelper\\.SetUpTest\n\n" + + "\\.\\.\\. Panic: WrongSetUpTestArgHelper\\.SetUpTest argument " + + "should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnWrongSetUpSuiteArg(c *C) { + helper := WrongSetUpSuiteArgHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(len(helper.calls), Equals, 0) + + expected := + "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongSetUpSuiteArgHelper\\.SetUpSuite\n\n" + + "\\.\\.\\. Panic: WrongSetUpSuiteArgHelper\\.SetUpSuite argument " + + "should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +// ----------------------------------------------------------------------- +// Nice errors also when tests or fixture have wrong arg count. + +func (s *FixtureS) TestPanicOnWrongTestArgCount(c *C) { + helper := WrongTestArgCountHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "TearDownTest") + c.Check(helper.calls[3], Equals, "SetUpTest") + c.Check(helper.calls[4], Equals, "Test2") + c.Check(helper.calls[5], Equals, "TearDownTest") + c.Check(helper.calls[6], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 7) + + expected := "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongTestArgCountHelper\\.Test1\n\n" + + "\\.\\.\\. Panic: WrongTestArgCountHelper\\.Test1 argument " + + "should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnWrongSetUpTestArgCount(c *C) { + helper := WrongSetUpTestArgCountHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(len(helper.calls), Equals, 0) + + expected := + "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongSetUpTestArgCountHelper\\.SetUpTest\n\n" + + "\\.\\.\\. Panic: WrongSetUpTestArgCountHelper\\.SetUpTest argument " + + "should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +func (s *FixtureS) TestPanicOnWrongSetUpSuiteArgCount(c *C) { + helper := WrongSetUpSuiteArgCountHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(len(helper.calls), Equals, 0) + + expected := + "^\n-+\n" + + "PANIC: fixture_test\\.go:[0-9]+: " + + "WrongSetUpSuiteArgCountHelper\\.SetUpSuite\n\n" + + "\\.\\.\\. Panic: WrongSetUpSuiteArgCountHelper" + + "\\.SetUpSuite argument should be \\*gocheck\\.C\n" + + c.Check(output.value, Matches, expected) +} + +// ----------------------------------------------------------------------- +// Helper test suites with wrong function arguments. + +type WrongTestArgHelper struct { + FixtureHelper +} + +func (s *WrongTestArgHelper) Test1(t int) { +} + +type WrongSetUpTestArgHelper struct { + FixtureHelper +} + +func (s *WrongSetUpTestArgHelper) SetUpTest(t int) { +} + +type WrongSetUpSuiteArgHelper struct { + FixtureHelper +} + +func (s *WrongSetUpSuiteArgHelper) SetUpSuite(t int) { +} + +type WrongTestArgCountHelper struct { + FixtureHelper +} + +func (s *WrongTestArgCountHelper) Test1(c *C, i int) { +} + +type WrongSetUpTestArgCountHelper struct { + FixtureHelper +} + +func (s *WrongSetUpTestArgCountHelper) SetUpTest(c *C, i int) { +} + +type WrongSetUpSuiteArgCountHelper struct { + FixtureHelper +} + +func (s *WrongSetUpSuiteArgCountHelper) SetUpSuite(c *C, i int) { +} + +// ----------------------------------------------------------------------- +// Ensure fixture doesn't run without tests. + +type NoTestsHelper struct { + hasRun bool +} + +func (s *NoTestsHelper) SetUpSuite(c *C) { + s.hasRun = true +} + +func (s *NoTestsHelper) TearDownSuite(c *C) { + s.hasRun = true +} + +func (s *FixtureS) TestFixtureDoesntRunWithoutTests(c *C) { + helper := NoTestsHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Check(helper.hasRun, Equals, false) +} + +// ----------------------------------------------------------------------- +// Verify that checks and assertions work correctly inside the fixture. + +type FixtureCheckHelper struct { + fail string + completed bool +} + +func (s *FixtureCheckHelper) SetUpSuite(c *C) { + switch s.fail { + case "SetUpSuiteAssert": + c.Assert(false, Equals, true) + case "SetUpSuiteCheck": + c.Check(false, Equals, true) + } + s.completed = true +} + +func (s *FixtureCheckHelper) SetUpTest(c *C) { + switch s.fail { + case "SetUpTestAssert": + c.Assert(false, Equals, true) + case "SetUpTestCheck": + c.Check(false, Equals, true) + } + s.completed = true +} + +func (s *FixtureCheckHelper) Test(c *C) { + // Do nothing. +} + +func (s *FixtureS) TestSetUpSuiteCheck(c *C) { + helper := FixtureCheckHelper{fail: "SetUpSuiteCheck"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Assert(output.value, Matches, + "\n---+\n"+ + "FAIL: fixture_test\\.go:[0-9]+: "+ + "FixtureCheckHelper\\.SetUpSuite\n\n"+ + "fixture_test\\.go:[0-9]+:\n"+ + " c\\.Check\\(false, Equals, true\\)\n"+ + "\\.+ obtained bool = false\n"+ + "\\.+ expected bool = true\n\n") + c.Assert(helper.completed, Equals, true) +} + +func (s *FixtureS) TestSetUpSuiteAssert(c *C) { + helper := FixtureCheckHelper{fail: "SetUpSuiteAssert"} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Assert(output.value, Matches, + "\n---+\n"+ + "FAIL: fixture_test\\.go:[0-9]+: "+ + "FixtureCheckHelper\\.SetUpSuite\n\n"+ + "fixture_test\\.go:[0-9]+:\n"+ + " c\\.Assert\\(false, Equals, true\\)\n"+ + "\\.+ obtained bool = false\n"+ + "\\.+ expected bool = true\n\n") + c.Assert(helper.completed, Equals, false) +} + +// ----------------------------------------------------------------------- +// Verify that logging within SetUpTest() persists within the test log itself. + +type FixtureLogHelper struct { + c *C +} + +func (s *FixtureLogHelper) SetUpTest(c *C) { + s.c = c + c.Log("1") +} + +func (s *FixtureLogHelper) Test(c *C) { + c.Log("2") + s.c.Log("3") + c.Log("4") + c.Fail() +} + +func (s *FixtureLogHelper) TearDownTest(c *C) { + s.c.Log("5") +} + +func (s *FixtureS) TestFixtureLogging(c *C) { + helper := FixtureLogHelper{} + output := String{} + Run(&helper, &RunConf{Output: &output}) + c.Assert(output.value, Matches, + "\n---+\n"+ + "FAIL: fixture_test\\.go:[0-9]+: "+ + "FixtureLogHelper\\.Test\n\n"+ + "1\n2\n3\n4\n5\n") +} + +// ----------------------------------------------------------------------- +// Skip() within fixture methods. + +func (s *FixtureS) TestSkipSuite(c *C) { + helper := FixtureHelper{skip: true, skipOnN: 0} + output := String{} + result := Run(&helper, &RunConf{Output: &output}) + c.Assert(output.value, Equals, "") + c.Assert(helper.calls[0], Equals, "SetUpSuite") + c.Assert(helper.calls[1], Equals, "TearDownSuite") + c.Assert(len(helper.calls), Equals, 2) + c.Assert(result.Skipped, Equals, 2) +} + +func (s *FixtureS) TestSkipTest(c *C) { + helper := FixtureHelper{skip: true, skipOnN: 1} + output := String{} + result := Run(&helper, &RunConf{Output: &output}) + c.Assert(helper.calls[0], Equals, "SetUpSuite") + c.Assert(helper.calls[1], Equals, "SetUpTest") + c.Assert(helper.calls[2], Equals, "SetUpTest") + c.Assert(helper.calls[3], Equals, "Test2") + c.Assert(helper.calls[4], Equals, "TearDownTest") + c.Assert(helper.calls[5], Equals, "TearDownSuite") + c.Assert(len(helper.calls), Equals, 6) + c.Assert(result.Skipped, Equals, 1) +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/foundation_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/foundation_test.go new file mode 100644 index 000000000..9f8915006 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/foundation_test.go @@ -0,0 +1,340 @@ +// These tests check that the foundations of gocheck are working properly. +// They already assume that fundamental failing is working already, though, +// since this was tested in bootstrap_test.go. Even then, some care may +// still have to be taken when using external functions, since they should +// of course not rely on functionality tested here. + +package gocheck_test + +import ( + "fmt" + "launchpad.net/gocheck" + "log" + "os" + "regexp" + "strings" +) + +// ----------------------------------------------------------------------- +// Foundation test suite. + +type FoundationS struct{} + +var foundationS = gocheck.Suite(&FoundationS{}) + +func (s *FoundationS) TestCountSuite(c *gocheck.C) { + suitesRun += 1 +} + +func (s *FoundationS) TestErrorf(c *gocheck.C) { + // Do not use checkState() here. It depends on Errorf() working. + expectedLog := fmt.Sprintf("foundation_test.go:%d:\n"+ + " c.Errorf(\"Error %%v!\", \"message\")\n"+ + "... Error: Error message!\n\n", + getMyLine()+1) + c.Errorf("Error %v!", "message") + failed := c.Failed() + c.Succeed() + if log := c.GetTestLog(); log != expectedLog { + c.Logf("Errorf() logged %#v rather than %#v", log, expectedLog) + c.Fail() + } + if !failed { + c.Logf("Errorf() didn't put the test in a failed state") + c.Fail() + } +} + +func (s *FoundationS) TestError(c *gocheck.C) { + expectedLog := fmt.Sprintf("foundation_test.go:%d:\n"+ + " c\\.Error\\(\"Error \", \"message!\"\\)\n"+ + "\\.\\.\\. Error: Error message!\n\n", + getMyLine()+1) + c.Error("Error ", "message!") + checkState(c, nil, + &expectedState{ + name: "Error(`Error `, `message!`)", + failed: true, + log: expectedLog, + }) +} + +func (s *FoundationS) TestFailNow(c *gocheck.C) { + defer (func() { + if !c.Failed() { + c.Error("FailNow() didn't fail the test") + } else { + c.Succeed() + if c.GetTestLog() != "" { + c.Error("Something got logged:\n" + c.GetTestLog()) + } + } + })() + + c.FailNow() + c.Log("FailNow() didn't stop the test") +} + +func (s *FoundationS) TestSucceedNow(c *gocheck.C) { + defer (func() { + if c.Failed() { + c.Error("SucceedNow() didn't succeed the test") + } + if c.GetTestLog() != "" { + c.Error("Something got logged:\n" + c.GetTestLog()) + } + })() + + c.Fail() + c.SucceedNow() + c.Log("SucceedNow() didn't stop the test") +} + +func (s *FoundationS) TestFailureHeader(c *gocheck.C) { + output := String{} + failHelper := FailHelper{} + gocheck.Run(&failHelper, &gocheck.RunConf{Output: &output}) + header := fmt.Sprintf(""+ + "\n-----------------------------------"+ + "-----------------------------------\n"+ + "FAIL: gocheck_test.go:%d: FailHelper.TestLogAndFail\n", + failHelper.testLine) + if strings.Index(output.value, header) == -1 { + c.Errorf(""+ + "Failure didn't print a proper header.\n"+ + "... Got:\n%s... Expected something with:\n%s", + output.value, header) + } +} + +func (s *FoundationS) TestFatal(c *gocheck.C) { + var line int + defer (func() { + if !c.Failed() { + c.Error("Fatal() didn't fail the test") + } else { + c.Succeed() + expected := fmt.Sprintf("foundation_test.go:%d:\n"+ + " c.Fatal(\"Die \", \"now!\")\n"+ + "... Error: Die now!\n\n", + line) + if c.GetTestLog() != expected { + c.Error("Incorrect log:", c.GetTestLog()) + } + } + })() + + line = getMyLine() + 1 + c.Fatal("Die ", "now!") + c.Log("Fatal() didn't stop the test") +} + +func (s *FoundationS) TestFatalf(c *gocheck.C) { + var line int + defer (func() { + if !c.Failed() { + c.Error("Fatalf() didn't fail the test") + } else { + c.Succeed() + expected := fmt.Sprintf("foundation_test.go:%d:\n"+ + " c.Fatalf(\"Die %%s!\", \"now\")\n"+ + "... Error: Die now!\n\n", + line) + if c.GetTestLog() != expected { + c.Error("Incorrect log:", c.GetTestLog()) + } + } + })() + + line = getMyLine() + 1 + c.Fatalf("Die %s!", "now") + c.Log("Fatalf() didn't stop the test") +} + +func (s *FoundationS) TestCallerLoggingInsideTest(c *gocheck.C) { + log := fmt.Sprintf(""+ + "foundation_test.go:%d:\n"+ + " result := c.Check\\(10, gocheck.Equals, 20\\)\n"+ + "\\.\\.\\. obtained int = 10\n"+ + "\\.\\.\\. expected int = 20\n\n", + getMyLine()+1) + result := c.Check(10, gocheck.Equals, 20) + checkState(c, result, + &expectedState{ + name: "Check(10, Equals, 20)", + result: false, + failed: true, + log: log, + }) +} + +func (s *FoundationS) TestCallerLoggingInDifferentFile(c *gocheck.C) { + result, line := checkEqualWrapper(c, 10, 20) + testLine := getMyLine() - 1 + log := fmt.Sprintf(""+ + "foundation_test.go:%d:\n"+ + " result, line := checkEqualWrapper\\(c, 10, 20\\)\n"+ + "gocheck_test.go:%d:\n"+ + " return c.Check\\(obtained, gocheck.Equals, expected\\), getMyLine\\(\\)\n"+ + "\\.\\.\\. obtained int = 10\n"+ + "\\.\\.\\. expected int = 20\n\n", + testLine, line) + checkState(c, result, + &expectedState{ + name: "Check(10, Equals, 20)", + result: false, + failed: true, + log: log, + }) +} + +// ----------------------------------------------------------------------- +// ExpectFailure() inverts the logic of failure. + +type ExpectFailureSucceedHelper struct{} + +func (s *ExpectFailureSucceedHelper) TestSucceed(c *gocheck.C) { + c.ExpectFailure("It booms!") + c.Error("Boom!") +} + +type ExpectFailureFailHelper struct{} + +func (s *ExpectFailureFailHelper) TestFail(c *gocheck.C) { + c.ExpectFailure("Bug #XYZ") +} + +func (s *FoundationS) TestExpectFailureFail(c *gocheck.C) { + helper := ExpectFailureFailHelper{} + output := String{} + result := gocheck.Run(&helper, &gocheck.RunConf{Output: &output}) + + expected := "" + + "^\n-+\n" + + "FAIL: foundation_test\\.go:[0-9]+:" + + " ExpectFailureFailHelper\\.TestFail\n\n" + + "\\.\\.\\. Error: Test succeeded, but was expected to fail\n" + + "\\.\\.\\. Reason: Bug #XYZ\n$" + + matched, err := regexp.MatchString(expected, output.value) + if err != nil { + c.Error("Bad expression: ", expected) + } else if !matched { + c.Error("ExpectFailure() didn't log properly:\n", output.value) + } + + c.Assert(result.ExpectedFailures, gocheck.Equals, 0) +} + +func (s *FoundationS) TestExpectFailureSucceed(c *gocheck.C) { + helper := ExpectFailureSucceedHelper{} + output := String{} + result := gocheck.Run(&helper, &gocheck.RunConf{Output: &output}) + + c.Assert(output.value, gocheck.Equals, "") + c.Assert(result.ExpectedFailures, gocheck.Equals, 1) +} + +func (s *FoundationS) TestExpectFailureSucceedVerbose(c *gocheck.C) { + helper := ExpectFailureSucceedHelper{} + output := String{} + result := gocheck.Run(&helper, &gocheck.RunConf{Output: &output, Verbose: true}) + + expected := "" + + "FAIL EXPECTED: foundation_test\\.go:[0-9]+:" + + " ExpectFailureSucceedHelper\\.TestSucceed \\(It booms!\\)\t *[.0-9]+s\n" + + matched, err := regexp.MatchString(expected, output.value) + if err != nil { + c.Error("Bad expression: ", expected) + } else if !matched { + c.Error("ExpectFailure() didn't log properly:\n", output.value) + } + + c.Assert(result.ExpectedFailures, gocheck.Equals, 1) +} + +// ----------------------------------------------------------------------- +// Skip() allows stopping a test without positive/negative results. + +type SkipTestHelper struct{} + +func (s *SkipTestHelper) TestFail(c *gocheck.C) { + c.Skip("Wrong platform or whatever") + c.Error("Boom!") +} + +func (s *FoundationS) TestSkip(c *gocheck.C) { + helper := SkipTestHelper{} + output := String{} + gocheck.Run(&helper, &gocheck.RunConf{Output: &output}) + + if output.value != "" { + c.Error("Skip() logged something:\n", output.value) + } +} + +func (s *FoundationS) TestSkipVerbose(c *gocheck.C) { + helper := SkipTestHelper{} + output := String{} + gocheck.Run(&helper, &gocheck.RunConf{Output: &output, Verbose: true}) + + expected := "SKIP: foundation_test\\.go:[0-9]+: SkipTestHelper\\.TestFail" + + " \\(Wrong platform or whatever\\)" + matched, err := regexp.MatchString(expected, output.value) + if err != nil { + c.Error("Bad expression: ", expected) + } else if !matched { + c.Error("Skip() didn't log properly:\n", output.value) + } +} + +// ----------------------------------------------------------------------- +// Check minimum *log.Logger interface provided by *gocheck.C. + +type minLogger interface { + Output(calldepth int, s string) error +} + +func (s *BootstrapS) TestMinLogger(c *gocheck.C) { + var logger minLogger + logger = log.New(os.Stderr, "", 0) + logger = c + logger.Output(0, "Hello there") + expected := "\\[LOG\\] [.0-9]+ Hello there\n" + output := c.GetTestLog() + matched, err := regexp.MatchString(expected, output) + if err != nil { + c.Error("Bad expression: ", expected) + } else if !matched { + c.Error("Output() didn't log properly:\n", output) + } +} + +// ----------------------------------------------------------------------- +// Ensure that suites with embedded types are working fine, including the +// the workaround for issue 906. + +type EmbeddedInternalS struct { + called bool +} + +type EmbeddedS struct { + EmbeddedInternalS +} + +var embeddedS = gocheck.Suite(&EmbeddedS{}) + +func (s *EmbeddedS) TestCountSuite(c *gocheck.C) { + suitesRun += 1 +} + +func (s *EmbeddedInternalS) TestMethod(c *gocheck.C) { + c.Error("TestMethod() of the embedded type was called!?") +} + +func (s *EmbeddedS) TestMethod(c *gocheck.C) { + // http://code.google.com/p/go/issues/detail?id=906 + c.Check(s.called, gocheck.Equals, false) // Go issue 906 is affecting the runner? + s.called = true +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/gocheck.go b/Godeps/_workspace/src/github.com/motain/gocheck/gocheck.go new file mode 100644 index 000000000..50af42e0a --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/gocheck.go @@ -0,0 +1,916 @@ +package gocheck + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +// ----------------------------------------------------------------------- +// Internal type which deals with suite method calling. + +const ( + fixtureKd = iota + testKd +) + +type funcKind int + +const ( + succeededSt = iota + failedSt + skippedSt + panickedSt + fixturePanickedSt + missedSt +) + +type funcStatus int + +// A method value can't reach its own Method structure. +type methodType struct { + reflect.Value + Info reflect.Method +} + +func newMethod(receiver reflect.Value, i int) *methodType { + return &methodType{receiver.Method(i), receiver.Type().Method(i)} +} + +func (method *methodType) PC() uintptr { + return method.Info.Func.Pointer() +} + +func (method *methodType) suiteName() string { + t := method.Info.Type.In(0) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t.Name() +} + +func (method *methodType) String() string { + return method.suiteName()+"."+method.Info.Name +} + +func (method *methodType) matches(re *regexp.Regexp) bool { + return (re.MatchString(method.Info.Name) || + re.MatchString(method.suiteName()) || + re.MatchString(method.String())) +} + +type C struct { + method *methodType + kind funcKind + status funcStatus + logb *logger + logw io.Writer + done chan *C + reason string + mustFail bool + tempDir *tempDir + timer +} + +func (c *C) stopNow() { + runtime.Goexit() +} + +// logger is a concurrency safe byte.Buffer +type logger struct { + sync.Mutex + writer bytes.Buffer +} + +func (l *logger) Write(buf []byte) (int, error) { + l.Lock() + defer l.Unlock() + return l.writer.Write(buf) +} + +func (l *logger) WriteTo(w io.Writer) (int64, error) { + l.Lock() + defer l.Unlock() + return l.writer.WriteTo(w) +} + +func (l *logger) String() string { + l.Lock() + defer l.Unlock() + return l.writer.String() +} + +// ----------------------------------------------------------------------- +// Handling of temporary files and directories. + +type tempDir struct { + sync.Mutex + _path string + _counter int +} + +func (td *tempDir) newPath() string { + td.Lock() + defer td.Unlock() + if td._path == "" { + var err error + for i := 0; i != 100; i++ { + path := fmt.Sprintf("%s/gocheck-%d", os.TempDir(), rand.Int()) + if err = os.Mkdir(path, 0700); err == nil { + td._path = path + break + } + } + if td._path == "" { + panic("Couldn't create temporary directory: " + err.Error()) + } + } + result := path.Join(td._path, strconv.Itoa(td._counter)) + td._counter += 1 + return result +} + +func (td *tempDir) removeAll() { + td.Lock() + defer td.Unlock() + if td._path != "" { + err := os.RemoveAll(td._path) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Error cleaning up temporaries: "+err.Error()) + } + } +} + +// Create a new temporary directory which is automatically removed after +// the suite finishes running. +func (c *C) MkDir() string { + path := c.tempDir.newPath() + if err := os.Mkdir(path, 0700); err != nil { + panic(fmt.Sprintf("Couldn't create temporary directory %s: %s", path, err.Error())) + } + return path +} + +// ----------------------------------------------------------------------- +// Low-level logging functions. + +func (c *C) log(args ...interface{}) { + c.writeLog([]byte(fmt.Sprint(args...) + "\n")) +} + +func (c *C) logf(format string, args ...interface{}) { + c.writeLog([]byte(fmt.Sprintf(format+"\n", args...))) +} + +func (c *C) logNewLine() { + c.writeLog([]byte{'\n'}) +} + +func (c *C) writeLog(buf []byte) { + c.logb.Write(buf) + if c.logw != nil { + c.logw.Write(buf) + } +} + +func hasStringOrError(x interface{}) (ok bool) { + _, ok = x.(fmt.Stringer) + if ok { + return + } + _, ok = x.(error) + return +} + +func (c *C) logValue(label string, value interface{}) { + if label == "" { + if hasStringOrError(value) { + c.logf("... %#v (%q)", value, value) + } else { + c.logf("... %#v", value) + } + } else if value == nil { + c.logf("... %s = nil", label) + } else { + if hasStringOrError(value) { + fv := fmt.Sprintf("%#v", value) + qv := fmt.Sprintf("%q", value) + if fv != qv { + c.logf("... %s %s = %s (%s)", label, reflect.TypeOf(value), fv, qv) + return + } + } + if s, ok := value.(string); ok && isMultiLine(s) { + c.logf(`... %s %s = "" +`, label, reflect.TypeOf(value)) + c.logMultiLine(s) + } else { + c.logf("... %s %s = %#v", label, reflect.TypeOf(value), value) + } + } +} + +func (c *C) logMultiLine(s string) { + b := make([]byte, 0, len(s)*2) + i := 0 + n := len(s) + for i < n { + j := i + 1 + for j < n && s[j-1] != '\n' { + j++ + } + b = append(b, "... "...) + b = strconv.AppendQuote(b, s[i:j]) + if j < n { + b = append(b, " +"...) + } + b = append(b, '\n') + i = j + } + c.writeLog(b) +} + +func isMultiLine(s string) bool { + for i := 0; i+1 < len(s); i++ { + if s[i] == '\n' { + return true + } + } + return false +} + +func (c *C) logString(issue string) { + c.log("... ", issue) +} + +func (c *C) logCaller(skip int) { + // This is a bit heavier than it ought to be. + skip += 1 // Our own frame. + pc, callerFile, callerLine, ok := runtime.Caller(skip) + if !ok { + return + } + var testFile string + var testLine int + testFunc := runtime.FuncForPC(c.method.PC()) + if runtime.FuncForPC(pc) != testFunc { + for { + skip += 1 + if pc, file, line, ok := runtime.Caller(skip); ok { + // Note that the test line may be different on + // distinct calls for the same test. Showing + // the "internal" line is helpful when debugging. + if runtime.FuncForPC(pc) == testFunc { + testFile, testLine = file, line + break + } + } else { + break + } + } + } + if testFile != "" && (testFile != callerFile || testLine != callerLine) { + c.logCode(testFile, testLine) + } + c.logCode(callerFile, callerLine) +} + +func (c *C) logCode(path string, line int) { + c.logf("%s:%d:", nicePath(path), line) + code, err := printLine(path, line) + if code == "" { + code = "..." // XXX Open the file and take the raw line. + if err != nil { + code += err.Error() + } + } + c.log(indent(code, " ")) +} + +var valueGo = filepath.Join("reflect", "value.go") + +func (c *C) logPanic(skip int, value interface{}) { + skip += 1 // Our own frame. + initialSkip := skip + for { + if pc, file, line, ok := runtime.Caller(skip); ok { + if skip == initialSkip { + c.logf("... Panic: %s (PC=0x%X)\n", value, pc) + } + name := niceFuncName(pc) + path := nicePath(file) + if name == "Value.call" && strings.HasSuffix(path, valueGo) { + break + } + c.logf("%s:%d\n in %s", nicePath(file), line, name) + } else { + break + } + skip += 1 + } +} + +func (c *C) logSoftPanic(issue string) { + c.log("... Panic: ", issue) +} + +func (c *C) logArgPanic(method *methodType, expectedType string) { + c.logf("... Panic: %s argument should be %s", + niceFuncName(method.PC()), expectedType) +} + +// ----------------------------------------------------------------------- +// Some simple formatting helpers. + +var initWD, initWDErr = os.Getwd() + +func init() { + if initWDErr == nil { + initWD = strings.Replace(initWD, "\\", "/", -1) + "/" + } +} + +func nicePath(path string) string { + if initWDErr == nil { + if strings.HasPrefix(path, initWD) { + return path[len(initWD):] + } + } + return path +} + +func niceFuncPath(pc uintptr) string { + function := runtime.FuncForPC(pc) + if function != nil { + filename, line := function.FileLine(pc) + return fmt.Sprintf("%s:%d", nicePath(filename), line) + } + return "" +} + +func niceFuncName(pc uintptr) string { + function := runtime.FuncForPC(pc) + if function != nil { + name := path.Base(function.Name()) + if i := strings.Index(name, "."); i > 0 { + name = name[i+1:] + } + if strings.HasPrefix(name, "(*") { + if i := strings.Index(name, ")"); i > 0 { + name = name[2:i] + name[i+1:] + } + } + if i := strings.LastIndex(name, ".*"); i != -1 { + name = name[:i] + "." + name[i+2:] + } + if i := strings.LastIndex(name, "·"); i != -1 { + name = name[:i] + "." + name[i+2:] + } + return name + } + return "" +} + +// ----------------------------------------------------------------------- +// Result tracker to aggregate call results. + +type Result struct { + Succeeded int + Failed int + Skipped int + Panicked int + FixturePanicked int + ExpectedFailures int + Missed int // Not even tried to run, related to a panic in the fixture. + RunError error // Houston, we've got a problem. +} + +type resultTracker struct { + result Result + _lastWasProblem bool + _waiting int + _missed int + _expectChan chan *C + _doneChan chan *C + _stopChan chan bool +} + +func newResultTracker() *resultTracker { + return &resultTracker{_expectChan: make(chan *C), // Synchronous + _doneChan: make(chan *C, 32), // Asynchronous + _stopChan: make(chan bool)} // Synchronous +} + +func (tracker *resultTracker) start() { + go tracker._loopRoutine() +} + +func (tracker *resultTracker) waitAndStop() { + <-tracker._stopChan +} + +func (tracker *resultTracker) expectCall(c *C) { + tracker._expectChan <- c +} + +func (tracker *resultTracker) callDone(c *C) { + tracker._doneChan <- c +} + +func (tracker *resultTracker) _loopRoutine() { + for { + var c *C + if tracker._waiting > 0 { + // Calls still running. Can't stop. + select { + // XXX Reindent this (not now to make diff clear) + case c = <-tracker._expectChan: + tracker._waiting += 1 + case c = <-tracker._doneChan: + tracker._waiting -= 1 + switch c.status { + case succeededSt: + if c.kind == testKd { + if c.mustFail { + tracker.result.ExpectedFailures++ + } else { + tracker.result.Succeeded++ + } + } + case failedSt: + tracker.result.Failed++ + case panickedSt: + if c.kind == fixtureKd { + tracker.result.FixturePanicked++ + } else { + tracker.result.Panicked++ + } + case fixturePanickedSt: + // Track it as missed, since the panic + // was on the fixture, not on the test. + tracker.result.Missed++ + case missedSt: + tracker.result.Missed++ + case skippedSt: + if c.kind == testKd { + tracker.result.Skipped++ + } + } + } + } else { + // No calls. Can stop, but no done calls here. + select { + case tracker._stopChan <- true: + return + case c = <-tracker._expectChan: + tracker._waiting += 1 + case c = <-tracker._doneChan: + panic("Tracker got an unexpected done call.") + } + } + } +} + +// ----------------------------------------------------------------------- +// The underlying suite runner. + +type suiteRunner struct { + suite interface{} + setUpSuite, tearDownSuite *methodType + setUpTest, tearDownTest *methodType + tests []*methodType + tracker *resultTracker + tempDir *tempDir + output *outputWriter + reportedProblemLast bool + benchTime time.Duration +} + +type RunConf struct { + Output io.Writer + Stream bool + Verbose bool + Filter string + Benchmark bool + BenchmarkTime time.Duration // Defaults to 1 second +} + +// Create a new suiteRunner able to run all methods in the given suite. +func newSuiteRunner(suite interface{}, runConf *RunConf) *suiteRunner { + var conf RunConf + if runConf != nil { + conf = *runConf + } + if conf.Output == nil { + conf.Output = os.Stdout + } + if conf.Benchmark { + conf.Verbose = true + } + + suiteType := reflect.TypeOf(suite) + suiteNumMethods := suiteType.NumMethod() + suiteValue := reflect.ValueOf(suite) + + runner := &suiteRunner{ + suite: suite, + output: newOutputWriter(conf.Output, conf.Stream, conf.Verbose), + tracker: newResultTracker(), + benchTime: conf.BenchmarkTime, + } + runner.tests = make([]*methodType, 0, suiteNumMethods) + runner.tempDir = new(tempDir) + if runner.benchTime == 0 { + runner.benchTime = 1 * time.Second + } + + var filterRegexp *regexp.Regexp + if conf.Filter != "" { + if regexp, err := regexp.Compile(conf.Filter); err != nil { + msg := "Bad filter expression: " + err.Error() + runner.tracker.result.RunError = errors.New(msg) + return runner + } else { + filterRegexp = regexp + } + } + + for i := 0; i != suiteNumMethods; i++ { + method := newMethod(suiteValue, i) + switch method.Info.Name { + case "SetUpSuite": + runner.setUpSuite = method + case "TearDownSuite": + runner.tearDownSuite = method + case "SetUpTest": + runner.setUpTest = method + case "TearDownTest": + runner.tearDownTest = method + default: + prefix := "Test" + if conf.Benchmark { + prefix = "Benchmark" + } + if !strings.HasPrefix(method.Info.Name, prefix) { + continue + } + if filterRegexp == nil || method.matches(filterRegexp) { + runner.tests = append(runner.tests, method) + } + } + } + return runner +} + +// Run all methods in the given suite. +func (runner *suiteRunner) run() *Result { + if runner.tracker.result.RunError == nil && len(runner.tests) > 0 { + runner.tracker.start() + if runner.checkFixtureArgs() { + c := runner.runFixture(runner.setUpSuite, nil) + if c == nil || c.status == succeededSt { + for i := 0; i != len(runner.tests); i++ { + c := runner.runTest(runner.tests[i]) + if c.status == fixturePanickedSt { + runner.skipTests(missedSt, runner.tests[i+1:]) + break + } + } + } else if c != nil && c.status == skippedSt { + runner.skipTests(skippedSt, runner.tests) + } else { + runner.skipTests(missedSt, runner.tests) + } + runner.runFixture(runner.tearDownSuite, nil) + } else { + runner.skipTests(missedSt, runner.tests) + } + runner.tracker.waitAndStop() + runner.tempDir.removeAll() + } + return &runner.tracker.result +} + +// Create a call object with the given suite method, and fork a +// goroutine with the provided dispatcher for running it. +func (runner *suiteRunner) forkCall(method *methodType, kind funcKind, logb *logger, dispatcher func(c *C)) *C { + var logw io.Writer + if runner.output.Stream { + logw = runner.output + } + if logb == nil { + logb = new(logger) + } + c := &C{ + method: method, + kind: kind, + logb: logb, + logw: logw, + tempDir: runner.tempDir, + done: make(chan *C, 1), + timer: timer{benchTime: runner.benchTime}, + } + runner.tracker.expectCall(c) + go (func() { + runner.reportCallStarted(c) + defer runner.callDone(c) + dispatcher(c) + })() + return c +} + +// Same as forkCall(), but wait for call to finish before returning. +func (runner *suiteRunner) runFunc(method *methodType, kind funcKind, logb *logger, dispatcher func(c *C)) *C { + c := runner.forkCall(method, kind, logb, dispatcher) + <-c.done + return c +} + +// Handle a finished call. If there were any panics, update the call status +// accordingly. Then, mark the call as done and report to the tracker. +func (runner *suiteRunner) callDone(c *C) { + value := recover() + if value != nil { + switch v := value.(type) { + case *fixturePanic: + if v.status == skippedSt { + c.status = skippedSt + } else { + c.logSoftPanic("Fixture has panicked (see related PANIC)") + c.status = fixturePanickedSt + } + default: + c.logPanic(1, value) + c.status = panickedSt + } + } + if c.mustFail { + switch c.status { + case failedSt: + c.status = succeededSt + case succeededSt: + c.status = failedSt + c.logString("Error: Test succeeded, but was expected to fail") + c.logString("Reason: " + c.reason) + } + } + + runner.reportCallDone(c) + c.done <- c +} + +// Runs a fixture call synchronously. The fixture will still be run in a +// goroutine like all suite methods, but this method will not return +// while the fixture goroutine is not done, because the fixture must be +// run in a desired order. +func (runner *suiteRunner) runFixture(method *methodType, logb *logger) *C { + if method != nil { + c := runner.runFunc(method, fixtureKd, logb, func(c *C) { + c.ResetTimer() + c.StartTimer() + defer c.StopTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + }) + return c + } + return nil +} + +// Run the fixture method with runFixture(), but panic with a fixturePanic{} +// in case the fixture method panics. This makes it easier to track the +// fixture panic together with other call panics within forkTest(). +func (runner *suiteRunner) runFixtureWithPanic(method *methodType, logb *logger, skipped *bool) *C { + if skipped != nil && *skipped { + return nil + } + c := runner.runFixture(method, logb) + if c != nil && c.status != succeededSt { + if skipped != nil { + *skipped = c.status == skippedSt + } + panic(&fixturePanic{c.status, method}) + } + return c +} + +type fixturePanic struct { + status funcStatus + method *methodType +} + +// Run the suite test method, together with the test-specific fixture, +// asynchronously. +func (runner *suiteRunner) forkTest(method *methodType) *C { + return runner.forkCall(method, testKd, nil, func(c *C) { + var skipped bool + defer runner.runFixtureWithPanic(runner.tearDownTest, nil, &skipped) + defer c.StopTimer() + benchN := 1 + for { + runner.runFixtureWithPanic(runner.setUpTest, c.logb, &skipped) + mt := c.method.Type() + if mt.NumIn() != 1 || mt.In(0) != reflect.TypeOf(c) { + // Rather than a plain panic, provide a more helpful message when + // the argument type is incorrect. + c.status = panickedSt + c.logArgPanic(c.method, "*gocheck.C") + return + } + if strings.HasPrefix(c.method.Info.Name, "Test") { + c.ResetTimer() + c.StartTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + return + } + if !strings.HasPrefix(c.method.Info.Name, "Benchmark") { + panic("unexpected method prefix: " + c.method.Info.Name) + } + + runtime.GC() + c.N = benchN + c.ResetTimer() + c.StartTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + c.StopTimer() + if c.status != succeededSt || c.duration >= c.benchTime || benchN >= 1e9 { + return + } + perOpN := int(1e9) + if c.nsPerOp() != 0 { + perOpN = int(c.benchTime.Nanoseconds() / c.nsPerOp()) + } + + // Logic taken from the stock testing package: + // - Run more iterations than we think we'll need for a second (1.5x). + // - Don't grow too fast in case we had timing errors previously. + // - Be sure to run at least one more than last time. + benchN = max(min(perOpN+perOpN/2, 100*benchN), benchN+1) + benchN = roundUp(benchN) + + skipped = true // Don't run the deferred one if this panics. + runner.runFixtureWithPanic(runner.tearDownTest, nil, nil) + skipped = false + } + }) +} + +// Same as forkTest(), but wait for the test to finish before returning. +func (runner *suiteRunner) runTest(method *methodType) *C { + c := runner.forkTest(method) + <-c.done + return c +} + +// Helper to mark tests as skipped or missed. A bit heavy for what +// it does, but it enables homogeneous handling of tracking, including +// nice verbose output. +func (runner *suiteRunner) skipTests(status funcStatus, methods []*methodType) { + for _, method := range methods { + runner.runFunc(method, testKd, nil, func(c *C) { + c.status = status + }) + } +} + +// Verify if the fixture arguments are *gocheck.C. In case of errors, +// log the error as a panic in the fixture method call, and return false. +func (runner *suiteRunner) checkFixtureArgs() bool { + succeeded := true + argType := reflect.TypeOf(&C{}) + for _, method := range []*methodType{runner.setUpSuite, runner.tearDownSuite, runner.setUpTest, runner.tearDownTest} { + if method != nil { + mt := method.Type() + if mt.NumIn() != 1 || mt.In(0) != argType { + succeeded = false + runner.runFunc(method, fixtureKd, nil, func(c *C) { + c.logArgPanic(method, "*gocheck.C") + c.status = panickedSt + }) + } + } + } + return succeeded +} + +func (runner *suiteRunner) reportCallStarted(c *C) { + runner.output.WriteCallStarted("START", c) +} + +func (runner *suiteRunner) reportCallDone(c *C) { + runner.tracker.callDone(c) + switch c.status { + case succeededSt: + if c.mustFail { + runner.output.WriteCallSuccess("FAIL EXPECTED", c) + } else { + runner.output.WriteCallSuccess("PASS", c) + } + case skippedSt: + runner.output.WriteCallSuccess("SKIP", c) + case failedSt: + runner.output.WriteCallProblem("FAIL", c) + case panickedSt: + runner.output.WriteCallProblem("PANIC", c) + case fixturePanickedSt: + // That's a testKd call reporting that its fixture + // has panicked. The fixture call which caused the + // panic itself was tracked above. We'll report to + // aid debugging. + runner.output.WriteCallProblem("PANIC", c) + case missedSt: + runner.output.WriteCallSuccess("MISS", c) + } +} + +// ----------------------------------------------------------------------- +// Output writer manages atomic output writing according to settings. + +type outputWriter struct { + m sync.Mutex + writer io.Writer + wroteCallProblemLast bool + Stream bool + Verbose bool +} + +func newOutputWriter(writer io.Writer, stream, verbose bool) *outputWriter { + return &outputWriter{writer: writer, Stream: stream, Verbose: verbose} +} + +func (ow *outputWriter) Write(content []byte) (n int, err error) { + ow.m.Lock() + n, err = ow.writer.Write(content) + ow.m.Unlock() + return +} + +func (ow *outputWriter) WriteCallStarted(label string, c *C) { + if ow.Stream { + header := renderCallHeader(label, c, "", "\n") + ow.m.Lock() + ow.writer.Write([]byte(header)) + ow.m.Unlock() + } +} + +func (ow *outputWriter) WriteCallProblem(label string, c *C) { + var prefix string + if !ow.Stream { + prefix = "\n-----------------------------------" + + "-----------------------------------\n" + } + header := renderCallHeader(label, c, prefix, "\n\n") + ow.m.Lock() + ow.wroteCallProblemLast = true + ow.writer.Write([]byte(header)) + if !ow.Stream { + c.logb.WriteTo(ow.writer) + } + ow.m.Unlock() +} + +func (ow *outputWriter) WriteCallSuccess(label string, c *C) { + if ow.Stream || (ow.Verbose && c.kind == testKd) { + // TODO Use a buffer here. + var suffix string + if c.reason != "" { + suffix = " (" + c.reason + ")" + } + if c.status == succeededSt { + suffix += "\t" + c.timerString() + } + suffix += "\n" + if ow.Stream { + suffix += "\n" + } + header := renderCallHeader(label, c, "", suffix) + ow.m.Lock() + // Resist temptation of using line as prefix above due to race. + if !ow.Stream && ow.wroteCallProblemLast { + header = "\n-----------------------------------" + + "-----------------------------------\n" + + header + } + ow.wroteCallProblemLast = false + ow.writer.Write([]byte(header)) + ow.m.Unlock() + } +} + +func renderCallHeader(label string, c *C, prefix, suffix string) string { + pc := c.method.PC() + return fmt.Sprintf("%s%s: %s: %s%s", prefix, label, niceFuncPath(pc), + niceFuncName(pc), suffix) +} + diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/gocheck_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/gocheck_test.go new file mode 100644 index 000000000..7575df4bb --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/gocheck_test.go @@ -0,0 +1,196 @@ +// This file contains just a few generic helpers which are used by the +// other test files. + +package gocheck_test + +import ( + "flag" + "fmt" + "launchpad.net/gocheck" + "os" + "regexp" + "runtime" + "testing" + "time" +) + +// We count the number of suites run at least to get a vague hint that the +// test suite is behaving as it should. Otherwise a bug introduced at the +// very core of the system could go unperceived. +const suitesRunExpected = 8 + +var suitesRun int = 0 + +func Test(t *testing.T) { + gocheck.TestingT(t) + if suitesRun != suitesRunExpected && flag.Lookup("gocheck.f").Value.String() == "" { + critical(fmt.Sprintf("Expected %d suites to run rather than %d", + suitesRunExpected, suitesRun)) + } +} + +// ----------------------------------------------------------------------- +// Helper functions. + +// Break down badly. This is used in test cases which can't yet assume +// that the fundamental bits are working. +func critical(error string) { + fmt.Fprintln(os.Stderr, "CRITICAL: "+error) + os.Exit(1) +} + +// Return the file line where it's called. +func getMyLine() int { + if _, _, line, ok := runtime.Caller(1); ok { + return line + } + return -1 +} + +// ----------------------------------------------------------------------- +// Helper type implementing a basic io.Writer for testing output. + +// Type implementing the io.Writer interface for analyzing output. +type String struct { + value string +} + +// The only function required by the io.Writer interface. Will append +// written data to the String.value string. +func (s *String) Write(p []byte) (n int, err error) { + s.value += string(p) + return len(p), nil +} + +// Trivial wrapper to test errors happening on a different file +// than the test itself. +func checkEqualWrapper(c *gocheck.C, obtained, expected interface{}) (result bool, line int) { + return c.Check(obtained, gocheck.Equals, expected), getMyLine() +} + +// ----------------------------------------------------------------------- +// Helper suite for testing basic fail behavior. + +type FailHelper struct { + testLine int +} + +func (s *FailHelper) TestLogAndFail(c *gocheck.C) { + s.testLine = getMyLine() - 1 + c.Log("Expected failure!") + c.Fail() +} + +// ----------------------------------------------------------------------- +// Helper suite for testing basic success behavior. + +type SuccessHelper struct{} + +func (s *SuccessHelper) TestLogAndSucceed(c *gocheck.C) { + c.Log("Expected success!") +} + +// ----------------------------------------------------------------------- +// Helper suite for testing ordering and behavior of fixture. + +type FixtureHelper struct { + calls []string + panicOn string + skip bool + skipOnN int + sleepOn string + sleep time.Duration + bytes int64 +} + +func (s *FixtureHelper) trace(name string, c *gocheck.C) { + s.calls = append(s.calls, name) + if name == s.panicOn { + panic(name) + } + if s.sleep > 0 && s.sleepOn == name { + time.Sleep(s.sleep) + } + if s.skip && s.skipOnN == len(s.calls)-1 { + c.Skip("skipOnN == n") + } +} + +func (s *FixtureHelper) SetUpSuite(c *gocheck.C) { + s.trace("SetUpSuite", c) +} + +func (s *FixtureHelper) TearDownSuite(c *gocheck.C) { + s.trace("TearDownSuite", c) +} + +func (s *FixtureHelper) SetUpTest(c *gocheck.C) { + s.trace("SetUpTest", c) +} + +func (s *FixtureHelper) TearDownTest(c *gocheck.C) { + s.trace("TearDownTest", c) +} + +func (s *FixtureHelper) Test1(c *gocheck.C) { + s.trace("Test1", c) +} + +func (s *FixtureHelper) Test2(c *gocheck.C) { + s.trace("Test2", c) +} + +func (s *FixtureHelper) Benchmark1(c *gocheck.C) { + s.trace("Benchmark1", c) + for i := 0; i < c.N; i++ { + time.Sleep(s.sleep) + } +} + +func (s *FixtureHelper) Benchmark2(c *gocheck.C) { + s.trace("Benchmark2", c) + c.SetBytes(1024) + for i := 0; i < c.N; i++ { + time.Sleep(s.sleep) + } +} + +// ----------------------------------------------------------------------- +// Helper which checks the state of the test and ensures that it matches +// the given expectations. Depends on c.Errorf() working, so shouldn't +// be used to test this one function. + +type expectedState struct { + name string + result interface{} + failed bool + log string +} + +// Verify the state of the test. Note that since this also verifies if +// the test is supposed to be in a failed state, no other checks should +// be done in addition to what is being tested. +func checkState(c *gocheck.C, result interface{}, expected *expectedState) { + failed := c.Failed() + c.Succeed() + log := c.GetTestLog() + matched, matchError := regexp.MatchString("^"+expected.log+"$", log) + if matchError != nil { + c.Errorf("Error in matching expression used in testing %s", + expected.name) + } else if !matched { + c.Errorf("%s logged:\n----------\n%s----------\n\nExpected:\n----------\n%s\n----------", + expected.name, log, expected.log) + } + if result != expected.result { + c.Errorf("%s returned %#v rather than %#v", + expected.name, result, expected.result) + } + if failed != expected.failed { + if failed { + c.Errorf("%s has failed when it shouldn't", expected.name) + } else { + c.Errorf("%s has not failed when it should", expected.name) + } + } +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/helpers.go b/Godeps/_workspace/src/github.com/motain/gocheck/helpers.go new file mode 100644 index 000000000..fe4ca5169 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/helpers.go @@ -0,0 +1,221 @@ +package gocheck + +import ( + "fmt" + "strings" + "time" +) + +// ----------------------------------------------------------------------- +// Basic succeeding/failing logic. + +// Return true if the currently running test has already failed. +func (c *C) Failed() bool { + return c.status == failedSt +} + +// Mark the currently running test as failed. Something ought to have been +// previously logged so that the developer knows what went wrong. The higher +// level helper functions will fail the test and do the logging properly. +func (c *C) Fail() { + c.status = failedSt +} + +// Mark the currently running test as failed, and stop running the test. +// Something ought to have been previously logged so that the developer +// knows what went wrong. The higher level helper functions will fail the +// test and do the logging properly. +func (c *C) FailNow() { + c.Fail() + c.stopNow() +} + +// Mark the currently running test as succeeded, undoing any previous +// failures. +func (c *C) Succeed() { + c.status = succeededSt +} + +// Mark the currently running test as succeeded, undoing any previous +// failures, and stop running the test. +func (c *C) SucceedNow() { + c.Succeed() + c.stopNow() +} + +// Expect the currently running test to fail, for the given reason. If the +// test does not fail, an error will be reported to raise the attention to +// this fact. The reason string is just a summary of why the given test is +// supposed to fail. This method is useful to temporarily disable tests +// which cover well known problems until a better time to fix the problem +// is found, without forgetting about the fact that a failure still exists. +func (c *C) ExpectFailure(reason string) { + if reason == "" { + panic("Missing reason why the test is expected to fail") + } + c.mustFail = true + c.reason = reason +} + +// Skip the running test, for the given reason. If used within SetUpTest, +// the individual test being set up will be skipped, and in SetUpSuite it +// will cause the whole suite to be skipped. +func (c *C) Skip(reason string) { + if reason == "" { + panic("Missing reason why the test is being skipped") + } + c.reason = reason + c.status = skippedSt + c.stopNow() +} + +// ----------------------------------------------------------------------- +// Basic logging. + +// Return the current test error output. +func (c *C) GetTestLog() string { + return c.logb.String() +} + +// Log some information into the test error output. The provided arguments +// will be assembled together into a string using fmt.Sprint(). +func (c *C) Log(args ...interface{}) { + c.log(args...) +} + +// Log some information into the test error output. The provided arguments +// will be assembled together into a string using fmt.Sprintf(). +func (c *C) Logf(format string, args ...interface{}) { + c.logf(format, args...) +} + +// Output enables *C to be used as a logger in functions that require only +// the minimum interface of *log.Logger. +func (c *C) Output(calldepth int, s string) error { + ns := time.Now().Sub(time.Time{}).Nanoseconds() + t := float64(ns%100e9) / 1e9 + c.Logf("[LOG] %.05f %s", t, s) + return nil +} + +// Log an error into the test error output, and mark the test as failed. +// The provided arguments will be assembled together into a string using +// fmt.Sprint(). +func (c *C) Error(args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) + c.logNewLine() + c.Fail() +} + +// Log an error into the test error output, and mark the test as failed. +// The provided arguments will be assembled together into a string using +// fmt.Sprintf(). +func (c *C) Errorf(format string, args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprintf("Error: "+format, args...)) + c.logNewLine() + c.Fail() +} + +// Log an error into the test error output, mark the test as failed, and +// stop the test execution. The provided arguments will be assembled +// together into a string using fmt.Sprint(). +func (c *C) Fatal(args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) + c.logNewLine() + c.FailNow() +} + +// Log an error into the test error output, mark the test as failed, and +// stop the test execution. The provided arguments will be assembled +// together into a string using fmt.Sprintf(). +func (c *C) Fatalf(format string, args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprintf(format, args...))) + c.logNewLine() + c.FailNow() +} + +// ----------------------------------------------------------------------- +// Generic checks and assertions based on checkers. + +// Verify if the first value matches with the expected value. What +// matching means is defined by the provided checker. In case they do not +// match, an error will be logged, the test will be marked as failed, and +// the test execution will continue. Some checkers may not need the expected +// argument (e.g. IsNil). In either case, any extra arguments provided to +// the function will be logged next to the reported problem when the +// matching fails. This is a handy way to provide problem-specific hints. +func (c *C) Check(obtained interface{}, checker Checker, args ...interface{}) bool { + return c.internalCheck("Check", obtained, checker, args...) +} + +// Ensure that the first value matches with the expected value. What +// matching means is defined by the provided checker. In case they do not +// match, an error will be logged, the test will be marked as failed, and +// the test execution will stop. Some checkers may not need the expected +// argument (e.g. IsNil). In either case, any extra arguments provided to +// the function will be logged next to the reported problem when the +// matching fails. This is a handy way to provide problem-specific hints. +func (c *C) Assert(obtained interface{}, checker Checker, args ...interface{}) { + if !c.internalCheck("Assert", obtained, checker, args...) { + c.stopNow() + } +} + +func (c *C) internalCheck(funcName string, obtained interface{}, checker Checker, args ...interface{}) bool { + if checker == nil { + c.logCaller(2) + c.logString(fmt.Sprintf("%s(obtained, nil!?, ...):", funcName)) + c.logString("Oops.. you've provided a nil checker!") + c.logNewLine() + c.Fail() + return false + } + + // If the last argument is a bug info, extract it out. + var comment CommentInterface + if len(args) > 0 { + if c, ok := args[len(args)-1].(CommentInterface); ok { + comment = c + args = args[:len(args)-1] + } + } + + params := append([]interface{}{obtained}, args...) + info := checker.Info() + + if len(params) != len(info.Params) { + names := append([]string{info.Params[0], info.Name}, info.Params[1:]...) + c.logCaller(2) + c.logString(fmt.Sprintf("%s(%s):", funcName, strings.Join(names, ", "))) + c.logString(fmt.Sprintf("Wrong number of parameters for %s: want %d, got %d", info.Name, len(names), len(params)+1)) + c.logNewLine() + c.Fail() + return false + } + + // Copy since it may be mutated by Check. + names := append([]string{}, info.Params...) + + // Do the actual check. + result, error := checker.Check(params, names) + if !result || error != "" { + c.logCaller(2) + for i := 0; i != len(params); i++ { + c.logValue(names[i], params[i]) + } + if comment != nil { + c.logString(comment.CheckCommentString()) + } + if error != "" { + c.logString(error) + } + c.logNewLine() + c.Fail() + return false + } + return true +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/helpers_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/helpers_test.go new file mode 100644 index 000000000..fb9ef8e18 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/helpers_test.go @@ -0,0 +1,491 @@ +// These tests verify the inner workings of the helper methods associated +// with gocheck.T. + +package gocheck_test + +import ( + "launchpad.net/gocheck" + "os" + "reflect" + "runtime" + "sync" +) + +var helpersS = gocheck.Suite(&HelpersS{}) + +type HelpersS struct{} + +func (s *HelpersS) TestCountSuite(c *gocheck.C) { + suitesRun += 1 +} + +// ----------------------------------------------------------------------- +// Fake checker and bug info to verify the behavior of Assert() and Check(). + +type MyChecker struct { + info *gocheck.CheckerInfo + params []interface{} + names []string + result bool + error string +} + +func (checker *MyChecker) Info() *gocheck.CheckerInfo { + if checker.info == nil { + return &gocheck.CheckerInfo{Name: "MyChecker", Params: []string{"myobtained", "myexpected"}} + } + return checker.info +} + +func (checker *MyChecker) Check(params []interface{}, names []string) (bool, string) { + rparams := checker.params + rnames := checker.names + checker.params = append([]interface{}{}, params...) + checker.names = append([]string{}, names...) + if rparams != nil { + copy(params, rparams) + } + if rnames != nil { + copy(names, rnames) + } + return checker.result, checker.error +} + +type myCommentType string + +func (c myCommentType) CheckCommentString() string { + return string(c) +} + +func myComment(s string) myCommentType { + return myCommentType(s) +} + +// ----------------------------------------------------------------------- +// Ensure a real checker actually works fine. + +func (s *HelpersS) TestCheckerInterface(c *gocheck.C) { + testHelperSuccess(c, "Check(1, Equals, 1)", true, func() interface{} { + return c.Check(1, gocheck.Equals, 1) + }) +} + +// ----------------------------------------------------------------------- +// Tests for Check(), mostly the same as for Assert() following these. + +func (s *HelpersS) TestCheckSucceedWithExpected(c *gocheck.C) { + checker := &MyChecker{result: true} + testHelperSuccess(c, "Check(1, checker, 2)", true, func() interface{} { + return c.Check(1, checker, 2) + }) + if !reflect.DeepEqual(checker.params, []interface{}{1, 2}) { + c.Fatalf("Bad params for check: %#v", checker.params) + } +} + +func (s *HelpersS) TestCheckSucceedWithoutExpected(c *gocheck.C) { + checker := &MyChecker{result: true, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + testHelperSuccess(c, "Check(1, checker)", true, func() interface{} { + return c.Check(1, checker) + }) + if !reflect.DeepEqual(checker.params, []interface{}{1}) { + c.Fatalf("Bad params for check: %#v", checker.params) + } +} + +func (s *HelpersS) TestCheckFailWithExpected(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, 2\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n\n" + testHelperFailure(c, "Check(1, checker, 2)", false, false, log, + func() interface{} { + return c.Check(1, checker, 2) + }) +} + +func (s *HelpersS) TestCheckFailWithExpectedAndComment(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, 2, myComment\\(\"Hello world!\"\\)\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n" + + "\\.+ Hello world!\n\n" + testHelperFailure(c, "Check(1, checker, 2, msg)", false, false, log, + func() interface{} { + return c.Check(1, checker, 2, myComment("Hello world!")) + }) +} + +func (s *HelpersS) TestCheckFailWithExpectedAndStaticComment(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " // Nice leading comment\\.\n" + + " return c\\.Check\\(1, checker, 2\\) // Hello there\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n\n" + testHelperFailure(c, "Check(1, checker, 2, msg)", false, false, log, + func() interface{} { + // Nice leading comment. + return c.Check(1, checker, 2) // Hello there + }) +} + +func (s *HelpersS) TestCheckFailWithoutExpected(c *gocheck.C) { + checker := &MyChecker{result: false, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker\\)\n" + + "\\.+ myvalue int = 1\n\n" + testHelperFailure(c, "Check(1, checker)", false, false, log, + func() interface{} { + return c.Check(1, checker) + }) +} + +func (s *HelpersS) TestCheckFailWithoutExpectedAndMessage(c *gocheck.C) { + checker := &MyChecker{result: false, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, myComment\\(\"Hello world!\"\\)\\)\n" + + "\\.+ myvalue int = 1\n" + + "\\.+ Hello world!\n\n" + testHelperFailure(c, "Check(1, checker, msg)", false, false, log, + func() interface{} { + return c.Check(1, checker, myComment("Hello world!")) + }) +} + +func (s *HelpersS) TestCheckWithMissingExpected(c *gocheck.C) { + checker := &MyChecker{result: true} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker\\)\n" + + "\\.+ Check\\(myobtained, MyChecker, myexpected\\):\n" + + "\\.+ Wrong number of parameters for MyChecker: " + + "want 3, got 2\n\n" + testHelperFailure(c, "Check(1, checker, !?)", false, false, log, + func() interface{} { + return c.Check(1, checker) + }) +} + +func (s *HelpersS) TestCheckWithTooManyExpected(c *gocheck.C) { + checker := &MyChecker{result: true} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, 2, 3\\)\n" + + "\\.+ Check\\(myobtained, MyChecker, myexpected\\):\n" + + "\\.+ Wrong number of parameters for MyChecker: " + + "want 3, got 4\n\n" + testHelperFailure(c, "Check(1, checker, 2, 3)", false, false, log, + func() interface{} { + return c.Check(1, checker, 2, 3) + }) +} + +func (s *HelpersS) TestCheckWithError(c *gocheck.C) { + checker := &MyChecker{result: false, error: "Some not so cool data provided!"} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, 2\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n" + + "\\.+ Some not so cool data provided!\n\n" + testHelperFailure(c, "Check(1, checker, 2)", false, false, log, + func() interface{} { + return c.Check(1, checker, 2) + }) +} + +func (s *HelpersS) TestCheckWithNilChecker(c *gocheck.C) { + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, nil\\)\n" + + "\\.+ Check\\(obtained, nil!\\?, \\.\\.\\.\\):\n" + + "\\.+ Oops\\.\\. you've provided a nil checker!\n\n" + testHelperFailure(c, "Check(obtained, nil)", false, false, log, + func() interface{} { + return c.Check(1, nil) + }) +} + +func (s *HelpersS) TestCheckWithParamsAndNamesMutation(c *gocheck.C) { + checker := &MyChecker{result: false, params: []interface{}{3, 4}, names: []string{"newobtained", "newexpected"}} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " return c\\.Check\\(1, checker, 2\\)\n" + + "\\.+ newobtained int = 3\n" + + "\\.+ newexpected int = 4\n\n" + testHelperFailure(c, "Check(1, checker, 2) with mutation", false, false, log, + func() interface{} { + return c.Check(1, checker, 2) + }) +} + +// ----------------------------------------------------------------------- +// Tests for Assert(), mostly the same as for Check() above. + +func (s *HelpersS) TestAssertSucceedWithExpected(c *gocheck.C) { + checker := &MyChecker{result: true} + testHelperSuccess(c, "Assert(1, checker, 2)", nil, func() interface{} { + c.Assert(1, checker, 2) + return nil + }) + if !reflect.DeepEqual(checker.params, []interface{}{1, 2}) { + c.Fatalf("Bad params for check: %#v", checker.params) + } +} + +func (s *HelpersS) TestAssertSucceedWithoutExpected(c *gocheck.C) { + checker := &MyChecker{result: true, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + testHelperSuccess(c, "Assert(1, checker)", nil, func() interface{} { + c.Assert(1, checker) + return nil + }) + if !reflect.DeepEqual(checker.params, []interface{}{1}) { + c.Fatalf("Bad params for check: %#v", checker.params) + } +} + +func (s *HelpersS) TestAssertFailWithExpected(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker, 2\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n\n" + testHelperFailure(c, "Assert(1, checker, 2)", nil, true, log, + func() interface{} { + c.Assert(1, checker, 2) + return nil + }) +} + +func (s *HelpersS) TestAssertFailWithExpectedAndMessage(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker, 2, myComment\\(\"Hello world!\"\\)\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n" + + "\\.+ Hello world!\n\n" + testHelperFailure(c, "Assert(1, checker, 2, msg)", nil, true, log, + func() interface{} { + c.Assert(1, checker, 2, myComment("Hello world!")) + return nil + }) +} + +func (s *HelpersS) TestAssertFailWithoutExpected(c *gocheck.C) { + checker := &MyChecker{result: false, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker\\)\n" + + "\\.+ myvalue int = 1\n\n" + testHelperFailure(c, "Assert(1, checker)", nil, true, log, + func() interface{} { + c.Assert(1, checker) + return nil + }) +} + +func (s *HelpersS) TestAssertFailWithoutExpectedAndMessage(c *gocheck.C) { + checker := &MyChecker{result: false, info: &gocheck.CheckerInfo{Params: []string{"myvalue"}}} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker, myComment\\(\"Hello world!\"\\)\\)\n" + + "\\.+ myvalue int = 1\n" + + "\\.+ Hello world!\n\n" + testHelperFailure(c, "Assert(1, checker, msg)", nil, true, log, + func() interface{} { + c.Assert(1, checker, myComment("Hello world!")) + return nil + }) +} + +func (s *HelpersS) TestAssertWithMissingExpected(c *gocheck.C) { + checker := &MyChecker{result: true} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker\\)\n" + + "\\.+ Assert\\(myobtained, MyChecker, myexpected\\):\n" + + "\\.+ Wrong number of parameters for MyChecker: " + + "want 3, got 2\n\n" + testHelperFailure(c, "Assert(1, checker, !?)", nil, true, log, + func() interface{} { + c.Assert(1, checker) + return nil + }) +} + +func (s *HelpersS) TestAssertWithError(c *gocheck.C) { + checker := &MyChecker{result: false, error: "Some not so cool data provided!"} + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, checker, 2\\)\n" + + "\\.+ myobtained int = 1\n" + + "\\.+ myexpected int = 2\n" + + "\\.+ Some not so cool data provided!\n\n" + testHelperFailure(c, "Assert(1, checker, 2)", nil, true, log, + func() interface{} { + c.Assert(1, checker, 2) + return nil + }) +} + +func (s *HelpersS) TestAssertWithNilChecker(c *gocheck.C) { + log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + + " c\\.Assert\\(1, nil\\)\n" + + "\\.+ Assert\\(obtained, nil!\\?, \\.\\.\\.\\):\n" + + "\\.+ Oops\\.\\. you've provided a nil checker!\n\n" + testHelperFailure(c, "Assert(obtained, nil)", nil, true, log, + func() interface{} { + c.Assert(1, nil) + return nil + }) +} + +// ----------------------------------------------------------------------- +// Ensure that values logged work properly in some interesting cases. + +func (s *HelpersS) TestValueLoggingWithArrays(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + + " return c\\.Check\\(\\[\\]byte{1, 2}, checker, \\[\\]byte{1, 3}\\)\n" + + "\\.+ myobtained \\[\\]uint8 = \\[\\]byte{0x1, 0x2}\n" + + "\\.+ myexpected \\[\\]uint8 = \\[\\]byte{0x1, 0x3}\n\n" + testHelperFailure(c, "Check([]byte{1}, chk, []byte{3})", false, false, log, + func() interface{} { + return c.Check([]byte{1, 2}, checker, []byte{1, 3}) + }) +} + +func (s *HelpersS) TestValueLoggingWithMultiLine(c *gocheck.C) { + checker := &MyChecker{result: false} + log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + + " return c\\.Check\\(\"a\\\\nb\\\\n\", checker, \"a\\\\nb\\\\nc\"\\)\n" + + "\\.+ myobtained string = \"\" \\+\n" + + "\\.+ \"a\\\\n\" \\+\n" + + "\\.+ \"b\\\\n\"\n" + + "\\.+ myexpected string = \"\" \\+\n" + + "\\.+ \"a\\\\n\" \\+\n" + + "\\.+ \"b\\\\n\" \\+\n" + + "\\.+ \"c\"\n\n" + testHelperFailure(c, `Check("a\nb\n", chk, "a\nb\nc")`, false, false, log, + func() interface{} { + return c.Check("a\nb\n", checker, "a\nb\nc") + }) +} + +func (s *HelpersS) TestValueLoggingWithMultiLineException(c *gocheck.C) { + // If the newline is at the end of the string, don't log as multi-line. + checker := &MyChecker{result: false} + log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + + " return c\\.Check\\(\"a b\\\\n\", checker, \"a\\\\nb\"\\)\n" + + "\\.+ myobtained string = \"a b\\\\n\"\n" + + "\\.+ myexpected string = \"\" \\+\n" + + "\\.+ \"a\\\\n\" \\+\n" + + "\\.+ \"b\"\n\n" + testHelperFailure(c, `Check("a b\n", chk, "a\nb")`, false, false, log, + func() interface{} { + return c.Check("a b\n", checker, "a\nb") + }) +} + +// ----------------------------------------------------------------------- +// MakeDir() tests. + +type MkDirHelper struct { + path1 string + path2 string + isDir1 bool + isDir2 bool + isDir3 bool + isDir4 bool +} + +func (s *MkDirHelper) SetUpSuite(c *gocheck.C) { + s.path1 = c.MkDir() + s.isDir1 = isDir(s.path1) +} + +func (s *MkDirHelper) Test(c *gocheck.C) { + s.path2 = c.MkDir() + s.isDir2 = isDir(s.path2) +} + +func (s *MkDirHelper) TearDownSuite(c *gocheck.C) { + s.isDir3 = isDir(s.path1) + s.isDir4 = isDir(s.path2) +} + +func (s *HelpersS) TestMkDir(c *gocheck.C) { + helper := MkDirHelper{} + output := String{} + gocheck.Run(&helper, &gocheck.RunConf{Output: &output}) + c.Assert(output.value, gocheck.Equals, "") + c.Check(helper.isDir1, gocheck.Equals, true) + c.Check(helper.isDir2, gocheck.Equals, true) + c.Check(helper.isDir3, gocheck.Equals, true) + c.Check(helper.isDir4, gocheck.Equals, true) + c.Check(helper.path1, gocheck.Not(gocheck.Equals), + helper.path2) + c.Check(isDir(helper.path1), gocheck.Equals, false) + c.Check(isDir(helper.path2), gocheck.Equals, false) +} + +func isDir(path string) bool { + if stat, err := os.Stat(path); err == nil { + return stat.IsDir() + } + return false +} + +// Concurrent logging should not corrupt the underling buffer. +// Use go test -race to detect the race in this test. +func (s *HelpersS) TestConcurrentLogging(c *gocheck.C) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) + var start, stop sync.WaitGroup + start.Add(1) + for i, n := 0, runtime.NumCPU()*2; i < n; i++ { + stop.Add(1) + go func(i int) { + start.Wait() + for j := 0; j < 30; j++ { + c.Logf("Worker %d: line %d", i, j) + } + stop.Done() + }(i) + } + start.Done() + stop.Wait() +} + +// ----------------------------------------------------------------------- +// A couple of helper functions to test helper functions. :-) + +func testHelperSuccess(c *gocheck.C, name string, expectedResult interface{}, closure func() interface{}) { + var result interface{} + defer (func() { + if err := recover(); err != nil { + panic(err) + } + checkState(c, result, + &expectedState{ + name: name, + result: expectedResult, + failed: false, + log: "", + }) + })() + result = closure() +} + +func testHelperFailure(c *gocheck.C, name string, expectedResult interface{}, shouldStop bool, log string, closure func() interface{}) { + var result interface{} + defer (func() { + if err := recover(); err != nil { + panic(err) + } + checkState(c, result, + &expectedState{ + name: name, + result: expectedResult, + failed: true, + log: log, + }) + })() + result = closure() + if shouldStop { + c.Logf("%s didn't stop when it should", name) + } +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/printer.go b/Godeps/_workspace/src/github.com/motain/gocheck/printer.go new file mode 100644 index 000000000..103ab14a0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/printer.go @@ -0,0 +1,168 @@ +package gocheck + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "os" +) + +func indent(s, with string) (r string) { + eol := true + for i := 0; i != len(s); i++ { + c := s[i] + switch { + case eol && c == '\n' || c == '\r': + case c == '\n' || c == '\r': + eol = true + case eol: + eol = false + s = s[:i] + with + s[i:] + i += len(with) + } + } + return s +} + +func printLine(filename string, line int) (string, error) { + fset := token.NewFileSet() + file, err := os.Open(filename) + if err != nil { + return "", err + } + fnode, err := parser.ParseFile(fset, filename, file, parser.ParseComments) + if err != nil { + return "", err + } + config := &printer.Config{Mode: printer.UseSpaces, Tabwidth: 4} + lp := &linePrinter{fset: fset, fnode: fnode, line: line, config: config} + ast.Walk(lp, fnode) + result := lp.output.Bytes() + // Comments leave \n at the end. + n := len(result) + for n > 0 && result[n-1] == '\n' { + n-- + } + return string(result[:n]), nil +} + +type linePrinter struct { + config *printer.Config + fset *token.FileSet + fnode *ast.File + line int + output bytes.Buffer + stmt ast.Stmt +} + +func (lp *linePrinter) emit() bool { + if lp.stmt != nil { + lp.trim(lp.stmt) + lp.printWithComments(lp.stmt) + lp.stmt = nil + return true + } + return false +} + +func (lp *linePrinter) printWithComments(n ast.Node) { + nfirst := lp.fset.Position(n.Pos()).Line + nlast := lp.fset.Position(n.End()).Line + for _, g := range lp.fnode.Comments { + cfirst := lp.fset.Position(g.Pos()).Line + clast := lp.fset.Position(g.End()).Line + if clast == nfirst-1 && lp.fset.Position(n.Pos()).Column == lp.fset.Position(g.Pos()).Column { + for _, c := range g.List { + lp.output.WriteString(c.Text) + lp.output.WriteByte('\n') + } + } + if cfirst >= nfirst && cfirst <= nlast && n.End() <= g.List[0].Slash { + // The printer will not include the comment if it starts past + // the node itself. Trick it into printing by overlapping the + // slash with the end of the statement. + g.List[0].Slash = n.End() - 1 + } + } + node := &printer.CommentedNode{n, lp.fnode.Comments} + lp.config.Fprint(&lp.output, lp.fset, node) +} + +func (lp *linePrinter) Visit(n ast.Node) (w ast.Visitor) { + if n == nil { + if lp.output.Len() == 0 { + lp.emit() + } + return nil + } + first := lp.fset.Position(n.Pos()).Line + last := lp.fset.Position(n.End()).Line + if first <= lp.line && last >= lp.line { + // Print the innermost statement containing the line. + if stmt, ok := n.(ast.Stmt); ok { + if _, ok := n.(*ast.BlockStmt); !ok { + lp.stmt = stmt + } + } + if first == lp.line && lp.emit() { + return nil + } + return lp + } + return nil +} + +func (lp *linePrinter) trim(n ast.Node) bool { + stmt, ok := n.(ast.Stmt) + if !ok { + return true + } + line := lp.fset.Position(n.Pos()).Line + if line != lp.line { + return false + } + switch stmt := stmt.(type) { + case *ast.IfStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.SwitchStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.TypeSwitchStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.CaseClause: + stmt.Body = lp.trimList(stmt.Body) + case *ast.CommClause: + stmt.Body = lp.trimList(stmt.Body) + case *ast.BlockStmt: + stmt.List = lp.trimList(stmt.List) + } + return true +} + +func (lp *linePrinter) trimBlock(stmt *ast.BlockStmt) *ast.BlockStmt { + if !lp.trim(stmt) { + return lp.emptyBlock(stmt) + } + stmt.Rbrace = stmt.Lbrace + return stmt +} + +func (lp *linePrinter) trimList(stmts []ast.Stmt) []ast.Stmt { + for i := 0; i != len(stmts); i++ { + if !lp.trim(stmts[i]) { + stmts[i] = lp.emptyStmt(stmts[i]) + break + } + } + return stmts +} + +func (lp *linePrinter) emptyStmt(n ast.Node) *ast.ExprStmt { + return &ast.ExprStmt{&ast.Ellipsis{n.Pos(), nil}} +} + +func (lp *linePrinter) emptyBlock(n ast.Node) *ast.BlockStmt { + p := n.Pos() + return &ast.BlockStmt{p, []ast.Stmt{lp.emptyStmt(n)}, p} +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/printer_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/printer_test.go new file mode 100644 index 000000000..547a92d85 --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/printer_test.go @@ -0,0 +1,104 @@ +package gocheck_test + +import ( + . "launchpad.net/gocheck" +) + +var _ = Suite(&PrinterS{}) + +type PrinterS struct{} + +func (s *PrinterS) TestCountSuite(c *C) { + suitesRun += 1 +} + +var printTestFuncLine int + +func init() { + printTestFuncLine = getMyLine() + 3 +} + +func printTestFunc() { + println(1) // Comment1 + if 2 == 2 { // Comment2 + println(3) // Comment3 + } + switch 5 { + case 6: println(6) // Comment6 + println(7) + } + switch interface{}(9).(type) {// Comment9 + case int: println(10) + println(11) + } + select { + case <-(chan bool)(nil): println(14) + println(15) + default: println(16) + println(17) + } + println(19, + 20) + _ = func() { println(21) + println(22) + } + println(24, func() { + println(25) + }) + // Leading comment + // with multiple lines. + println(29) // Comment29 +} + +var printLineTests = []struct { + line int + output string +}{ + {1, "println(1) // Comment1"}, + {2, "if 2 == 2 { // Comment2\n ...\n}"}, + {3, "println(3) // Comment3"}, + {5, "switch 5 {\n...\n}"}, + {6, "case 6:\n println(6) // Comment6\n ..."}, + {7, "println(7)"}, + {9, "switch interface{}(9).(type) { // Comment9\n...\n}"}, + {10, "case int:\n println(10)\n ..."}, + {14, "case <-(chan bool)(nil):\n println(14)\n ..."}, + {15, "println(15)"}, + {16, "default:\n println(16)\n ..."}, + {17, "println(17)"}, + {19, "println(19,\n 20)"}, + {20, "println(19,\n 20)"}, + {21, "_ = func() {\n println(21)\n println(22)\n}"}, + {22, "println(22)"}, + {24, "println(24, func() {\n println(25)\n})"}, + {25, "println(25)"}, + {26, "println(24, func() {\n println(25)\n})"}, + {29, "// Leading comment\n// with multiple lines.\nprintln(29) // Comment29"}, +} + +func (s *PrinterS) TestPrintLine(c *C) { + for _, test := range printLineTests { + output, err := PrintLine("printer_test.go", printTestFuncLine+test.line) + c.Assert(err, IsNil) + c.Assert(output, Equals, test.output) + } +} + +var indentTests = []struct { + in, out string +}{ + {"", ""}, + {"\n", "\n"}, + {"a", ">>>a"}, + {"a\n", ">>>a\n"}, + {"a\nb", ">>>a\n>>>b"}, + {" ", ">>> "}, +} + +func (s *PrinterS) TestIndent(c *C) { + for _, test := range indentTests { + out := Indent(test.in, ">>>") + c.Assert(out, Equals, test.out) + } + +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/run.go b/Godeps/_workspace/src/github.com/motain/gocheck/run.go new file mode 100644 index 000000000..dc75d772b --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/run.go @@ -0,0 +1,152 @@ +package gocheck + +import ( + "bufio" + "flag" + "fmt" + "os" + "testing" + "time" +) + +// ----------------------------------------------------------------------- +// Test suite registry. + +var allSuites []interface{} + +// Register the given value as a test suite to be run. Any methods starting +// with the Test prefix in the given value will be considered as a test to +// be run. +func Suite(suite interface{}) interface{} { + allSuites = append(allSuites, suite) + return suite +} + +// ----------------------------------------------------------------------- +// Public running interface. + +var ( + filterFlag = flag.String("gocheck.f", "", "Regular expression selecting which tests and/or suites to run") + verboseFlag = flag.Bool("gocheck.v", false, "Verbose mode") + streamFlag = flag.Bool("gocheck.vv", false, "Super verbose mode (disables output caching)") + benchFlag = flag.Bool("gocheck.b", false, "Run benchmarks") + benchTime = flag.Duration("gocheck.btime", 1 * time.Second, "approximate run time for each benchmark") + listFlag = flag.Bool("gocheck.list", false, "List the names of all tests that will be run") +) + +// Run all test suites registered with the Suite() function, printing +// results to stdout, and reporting any failures back to the 'testing' +// module. +func TestingT(testingT *testing.T) { + conf := &RunConf{ + Filter: *filterFlag, + Verbose: *verboseFlag, + Stream: *streamFlag, + Benchmark: *benchFlag, + BenchmarkTime: *benchTime, + } + if *listFlag { + w := bufio.NewWriter(os.Stdout) + for _, name := range ListAll(conf) { + fmt.Fprintln(w, name) + } + w.Flush() + return + } + result := RunAll(conf) + println(result.String()) + if !result.Passed() { + testingT.Fail() + } +} + +// RunAll runs all test suites registered with the Suite() function, using the +// given run configuration. +func RunAll(runConf *RunConf) *Result { + result := Result{} + for _, suite := range allSuites { + result.Add(Run(suite, runConf)) + } + return &result +} + +// Run runs the given test suite using the provided run configuration. +func Run(suite interface{}, runConf *RunConf) *Result { + runner := newSuiteRunner(suite, runConf) + return runner.run() +} + +// ListAll returns the names of all the test functions registered with the +// Suite function that will be run with the provided run configuration. +func ListAll(runConf *RunConf) []string { + var names []string + for _, suite := range allSuites { + names = append(names, List(suite, runConf)...) + } + return names +} + +// List prints the names of the test functions in the given +// suite that will be run with the provided run configuration +// to the given Writer. +func List(suite interface{}, runConf *RunConf) []string { + var names []string + runner := newSuiteRunner(suite, runConf) + for _, t := range runner.tests { + names = append(names, t.String()) + } + return names +} + +// ----------------------------------------------------------------------- +// Result methods. + +func (r *Result) Add(other *Result) { + r.Succeeded += other.Succeeded + r.Skipped += other.Skipped + r.Failed += other.Failed + r.Panicked += other.Panicked + r.FixturePanicked += other.FixturePanicked + r.ExpectedFailures += other.ExpectedFailures + r.Missed += other.Missed +} + +func (r *Result) Passed() bool { + return (r.Failed == 0 && r.Panicked == 0 && + r.FixturePanicked == 0 && r.Missed == 0 && + r.RunError == nil) +} + +func (r *Result) String() string { + if r.RunError != nil { + return "ERROR: " + r.RunError.Error() + } + + var value string + if r.Failed == 0 && r.Panicked == 0 && r.FixturePanicked == 0 && + r.Missed == 0 { + value = "OK: " + } else { + value = "OOPS: " + } + value += fmt.Sprintf("%d passed", r.Succeeded) + if r.Skipped != 0 { + value += fmt.Sprintf(", %d skipped", r.Skipped) + } + if r.ExpectedFailures != 0 { + value += fmt.Sprintf(", %d expected failures", r.ExpectedFailures) + } + if r.Failed != 0 { + value += fmt.Sprintf(", %d FAILED", r.Failed) + } + if r.Panicked != 0 { + value += fmt.Sprintf(", %d PANICKED", r.Panicked) + } + if r.FixturePanicked != 0 { + value += fmt.Sprintf(", %d FIXTURE-PANICKED", r.FixturePanicked) + } + if r.Missed != 0 { + value += fmt.Sprintf(", %d MISSED", r.Missed) + } + return value +} diff --git a/Godeps/_workspace/src/github.com/motain/gocheck/run_test.go b/Godeps/_workspace/src/github.com/motain/gocheck/run_test.go new file mode 100644 index 000000000..6793e4cdb --- /dev/null +++ b/Godeps/_workspace/src/github.com/motain/gocheck/run_test.go @@ -0,0 +1,398 @@ +// These tests verify the test running logic. + +package gocheck_test + + +import ( + "errors" + . "launchpad.net/gocheck" + "sync" +) + +var runnerS = Suite(&RunS{}) + +type RunS struct{} + +func (s *RunS) TestCountSuite(c *C) { + suitesRun += 1 +} + +// ----------------------------------------------------------------------- +// Tests ensuring result counting works properly. + +func (s *RunS) TestSuccess(c *C) { + output := String{} + result := Run(&SuccessHelper{}, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 1) + c.Check(result.Failed, Equals, 0) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 0) + c.Check(result.FixturePanicked, Equals, 0) + c.Check(result.Missed, Equals, 0) + c.Check(result.RunError, IsNil) +} + +func (s *RunS) TestFailure(c *C) { + output := String{} + result := Run(&FailHelper{}, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 0) + c.Check(result.Failed, Equals, 1) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 0) + c.Check(result.FixturePanicked, Equals, 0) + c.Check(result.Missed, Equals, 0) + c.Check(result.RunError, IsNil) +} + +func (s *RunS) TestFixture(c *C) { + output := String{} + result := Run(&FixtureHelper{}, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 2) + c.Check(result.Failed, Equals, 0) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 0) + c.Check(result.FixturePanicked, Equals, 0) + c.Check(result.Missed, Equals, 0) + c.Check(result.RunError, IsNil) +} + +func (s *RunS) TestPanicOnTest(c *C) { + output := String{} + helper := &FixtureHelper{panicOn: "Test1"} + result := Run(helper, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 1) + c.Check(result.Failed, Equals, 0) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 1) + c.Check(result.FixturePanicked, Equals, 0) + c.Check(result.Missed, Equals, 0) + c.Check(result.RunError, IsNil) +} + +func (s *RunS) TestPanicOnSetUpTest(c *C) { + output := String{} + helper := &FixtureHelper{panicOn: "SetUpTest"} + result := Run(helper, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 0) + c.Check(result.Failed, Equals, 0) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 0) + c.Check(result.FixturePanicked, Equals, 1) + c.Check(result.Missed, Equals, 2) + c.Check(result.RunError, IsNil) +} + +func (s *RunS) TestPanicOnSetUpSuite(c *C) { + output := String{} + helper := &FixtureHelper{panicOn: "SetUpSuite"} + result := Run(helper, &RunConf{Output: &output}) + c.Check(result.Succeeded, Equals, 0) + c.Check(result.Failed, Equals, 0) + c.Check(result.Skipped, Equals, 0) + c.Check(result.Panicked, Equals, 0) + c.Check(result.FixturePanicked, Equals, 1) + c.Check(result.Missed, Equals, 2) + c.Check(result.RunError, IsNil) +} + +// ----------------------------------------------------------------------- +// Check result aggregation. + +func (s *RunS) TestAdd(c *C) { + result := &Result{ + Succeeded: 1, + Skipped: 2, + Failed: 3, + Panicked: 4, + FixturePanicked: 5, + Missed: 6, + ExpectedFailures: 7, + } + result.Add(&Result{ + Succeeded: 10, + Skipped: 20, + Failed: 30, + Panicked: 40, + FixturePanicked: 50, + Missed: 60, + ExpectedFailures: 70, + }) + c.Check(result.Succeeded, Equals, 11) + c.Check(result.Skipped, Equals, 22) + c.Check(result.Failed, Equals, 33) + c.Check(result.Panicked, Equals, 44) + c.Check(result.FixturePanicked, Equals, 55) + c.Check(result.Missed, Equals, 66) + c.Check(result.ExpectedFailures, Equals, 77) + c.Check(result.RunError, IsNil) +} + +// ----------------------------------------------------------------------- +// Check the Passed() method. + +func (s *RunS) TestPassed(c *C) { + c.Assert((&Result{}).Passed(), Equals, true) + c.Assert((&Result{Succeeded: 1}).Passed(), Equals, true) + c.Assert((&Result{Skipped: 1}).Passed(), Equals, true) + c.Assert((&Result{Failed: 1}).Passed(), Equals, false) + c.Assert((&Result{Panicked: 1}).Passed(), Equals, false) + c.Assert((&Result{FixturePanicked: 1}).Passed(), Equals, false) + c.Assert((&Result{Missed: 1}).Passed(), Equals, false) + c.Assert((&Result{RunError: errors.New("!")}).Passed(), Equals, false) +} + +// ----------------------------------------------------------------------- +// Check that result printing is working correctly. + +func (s *RunS) TestPrintSuccess(c *C) { + result := &Result{Succeeded: 5} + c.Check(result.String(), Equals, "OK: 5 passed") +} + +func (s *RunS) TestPrintFailure(c *C) { + result := &Result{Failed: 5} + c.Check(result.String(), Equals, "OOPS: 0 passed, 5 FAILED") +} + +func (s *RunS) TestPrintSkipped(c *C) { + result := &Result{Skipped: 5} + c.Check(result.String(), Equals, "OK: 0 passed, 5 skipped") +} + +func (s *RunS) TestPrintExpectedFailures(c *C) { + result := &Result{ExpectedFailures: 5} + c.Check(result.String(), Equals, "OK: 0 passed, 5 expected failures") +} + +func (s *RunS) TestPrintPanicked(c *C) { + result := &Result{Panicked: 5} + c.Check(result.String(), Equals, "OOPS: 0 passed, 5 PANICKED") +} + +func (s *RunS) TestPrintFixturePanicked(c *C) { + result := &Result{FixturePanicked: 5} + c.Check(result.String(), Equals, "OOPS: 0 passed, 5 FIXTURE-PANICKED") +} + +func (s *RunS) TestPrintMissed(c *C) { + result := &Result{Missed: 5} + c.Check(result.String(), Equals, "OOPS: 0 passed, 5 MISSED") +} + +func (s *RunS) TestPrintAll(c *C) { + result := &Result{Succeeded: 1, Skipped: 2, ExpectedFailures: 3, + Panicked: 4, FixturePanicked: 5, Missed: 6} + c.Check(result.String(), Equals, + "OOPS: 1 passed, 2 skipped, 3 expected failures, 4 PANICKED, "+ + "5 FIXTURE-PANICKED, 6 MISSED") +} + +func (s *RunS) TestPrintRunError(c *C) { + result := &Result{Succeeded: 1, Failed: 1, + RunError: errors.New("Kaboom!")} + c.Check(result.String(), Equals, "ERROR: Kaboom!") +} + +// ----------------------------------------------------------------------- +// Verify that the method pattern flag works correctly. + +func (s *RunS) TestFilterTestName(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "Test[91]"} + Run(&helper, &runConf) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 5) +} + +func (s *RunS) TestFilterTestNameWithAll(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: ".*"} + Run(&helper, &runConf) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Test2") + c.Check(helper.calls[6], Equals, "TearDownTest") + c.Check(helper.calls[7], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 8) +} + +func (s *RunS) TestFilterSuiteName(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "FixtureHelper"} + Run(&helper, &runConf) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test1") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "SetUpTest") + c.Check(helper.calls[5], Equals, "Test2") + c.Check(helper.calls[6], Equals, "TearDownTest") + c.Check(helper.calls[7], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 8) +} + +func (s *RunS) TestFilterSuiteNameAndTestName(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "FixtureHelper\\.Test2"} + Run(&helper, &runConf) + c.Check(helper.calls[0], Equals, "SetUpSuite") + c.Check(helper.calls[1], Equals, "SetUpTest") + c.Check(helper.calls[2], Equals, "Test2") + c.Check(helper.calls[3], Equals, "TearDownTest") + c.Check(helper.calls[4], Equals, "TearDownSuite") + c.Check(len(helper.calls), Equals, 5) +} + +func (s *RunS) TestFilterAllOut(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "NotFound"} + Run(&helper, &runConf) + c.Check(len(helper.calls), Equals, 0) +} + +func (s *RunS) TestRequirePartialMatch(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "est"} + Run(&helper, &runConf) + c.Check(len(helper.calls), Equals, 8) +} + +func (s *RunS) TestFilterError(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Filter: "]["} + result := Run(&helper, &runConf) + c.Check(result.String(), Equals, + "ERROR: Bad filter expression: error parsing regexp: missing closing ]: `[`") + c.Check(len(helper.calls), Equals, 0) +} + +// ----------------------------------------------------------------------- +// Verify that List works correctly. + +func (s *RunS) TestListFiltered(c *C) { + names := List(&FixtureHelper{}, &RunConf{Filter: "1"}) + c.Assert(names, DeepEquals, []string{ + "FixtureHelper.Test1", + }) +} + +func (s *RunS) TestList(c *C) { + names := List(&FixtureHelper{}, &RunConf{}) + c.Assert(names, DeepEquals, []string{ + "FixtureHelper.Test1", + "FixtureHelper.Test2", + }) +} + +// ----------------------------------------------------------------------- +// Verify that verbose mode prints tests which pass as well. + +func (s *RunS) TestVerboseMode(c *C) { + helper := FixtureHelper{} + output := String{} + runConf := RunConf{Output: &output, Verbose: true} + Run(&helper, &runConf) + + expected := "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Test1\t *[.0-9]+s\n" + + "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Test2\t *[.0-9]+s\n" + + c.Assert(output.value, Matches, expected) +} + +func (s *RunS) TestVerboseModeWithFailBeforePass(c *C) { + helper := FixtureHelper{panicOn: "Test1"} + output := String{} + runConf := RunConf{Output: &output, Verbose: true} + Run(&helper, &runConf) + + expected := "(?s).*PANIC.*\n-+\n" + // Should have an extra line. + "PASS: gocheck_test\\.go:[0-9]+: FixtureHelper\\.Test2\t *[.0-9]+s\n" + + c.Assert(output.value, Matches, expected) +} + +// ----------------------------------------------------------------------- +// Verify the stream output mode. In this mode there's no output caching. + +type StreamHelper struct { + l2 sync.Mutex + l3 sync.Mutex +} + +func (s *StreamHelper) SetUpSuite(c *C) { + c.Log("0") +} + +func (s *StreamHelper) Test1(c *C) { + c.Log("1") + s.l2.Lock() + s.l3.Lock() + go func() { + s.l2.Lock() // Wait for "2". + c.Log("3") + s.l3.Unlock() + }() +} + +func (s *StreamHelper) Test2(c *C) { + c.Log("2") + s.l2.Unlock() + s.l3.Lock() // Wait for "3". + c.Fail() + c.Log("4") +} + +func (s *RunS) TestStreamMode(c *C) { + helper := &StreamHelper{} + output := String{} + runConf := RunConf{Output: &output, Stream: true} + Run(helper, &runConf) + + expected := "START: run_test\\.go:[0-9]+: StreamHelper\\.SetUpSuite\n0\n" + + "PASS: run_test\\.go:[0-9]+: StreamHelper\\.SetUpSuite\t *[.0-9]+s\n\n" + + "START: run_test\\.go:[0-9]+: StreamHelper\\.Test1\n1\n" + + "PASS: run_test\\.go:[0-9]+: StreamHelper\\.Test1\t *[.0-9]+s\n\n" + + "START: run_test\\.go:[0-9]+: StreamHelper\\.Test2\n2\n3\n4\n" + + "FAIL: run_test\\.go:[0-9]+: StreamHelper\\.Test2\n\n" + + c.Assert(output.value, Matches, expected) +} + +type StreamMissHelper struct{} + +func (s *StreamMissHelper) SetUpSuite(c *C) { + c.Log("0") + c.Fail() +} + +func (s *StreamMissHelper) Test1(c *C) { + c.Log("1") +} + +func (s *RunS) TestStreamModeWithMiss(c *C) { + helper := &StreamMissHelper{} + output := String{} + runConf := RunConf{Output: &output, Stream: true} + Run(helper, &runConf) + + expected := "START: run_test\\.go:[0-9]+: StreamMissHelper\\.SetUpSuite\n0\n" + + "FAIL: run_test\\.go:[0-9]+: StreamMissHelper\\.SetUpSuite\n\n" + + "START: run_test\\.go:[0-9]+: StreamMissHelper\\.Test1\n" + + "MISS: run_test\\.go:[0-9]+: StreamMissHelper\\.Test1\n\n" + + c.Assert(output.value, Matches, expected) +} From 4aa75994b0851bfd8ba9bafef773291689fe35ac Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Fri, 15 May 2015 17:29:48 -0500 Subject: [PATCH 06/13] Add ability to open bucket using S3 object for testing purposes. --- backend/s3/s3.go | 5 +++++ backend/s3_test.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 48f7beaa6..40f157179 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -28,6 +28,11 @@ type S3 struct { path string } +// Open a backend using an S3 bucket object +func OpenS3Bucket(bucket *s3.Bucket, bucketname string) *S3 { + return &S3{bucket: bucket, path: bucketname} +} + // Open opens the s3 backend at bucket and region. func Open(regionname, bucketname string) (*S3, error) { auth, err := aws.EnvAuth() diff --git a/backend/s3_test.go b/backend/s3_test.go index 84b4d6d74..68cf8a079 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -23,7 +23,7 @@ func setupS3Backend(t *testing.T) *bes3.S3 { t.Logf("created s3 backend locally at %s", testServer.URL) - return bes3.S3{bucket: bucket, path: "testbucket"} + return bes3.OpenS3Bucket(bucket, "testbucket") } func teardownS3Backend(t *testing.T, b *bes3.S3) { From 2350419f59692b8c21ef7124455bcb4aed2b498f Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sat, 16 May 2015 06:48:55 -0500 Subject: [PATCH 07/13] Modify test suite to use s3 test server. Fix s3 backend to return error on attempting to write to already existing key. --- backend/s3/s3.go | 9 ++++++++- backend/s3_test.go | 44 +++++++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 40f157179..991385525 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -87,8 +87,15 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { bb.final = true path := s3path(t, name) + + // Check key does not already exist + key, err := bb.b.bucket.GetKey(path) + if err == nil && key.Key == path { + return errors.New("key already exists!") + } + bb.b.mput.Lock() - err := bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") + err = bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") bb.b.mput.Unlock() bb.buf.Reset() return err diff --git a/backend/s3_test.go b/backend/s3_test.go index 68cf8a079..64a23b9a1 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -5,39 +5,49 @@ import ( "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/s3" - "github.com/mitchellh/goamz/testutil" + "github.com/mitchellh/goamz/s3/s3test" bes3 "github.com/restic/restic/backend/s3" . "github.com/restic/restic/test" ) -var testServer = testutil.NewHTTPServer() +type LocalServer struct { + auth aws.Auth + region aws.Region + srv *s3test.Server + config *s3test.Config +} + +var s LocalServer func setupS3Backend(t *testing.T) *bes3.S3 { - testServer.Start() - auth := aws.Auth{"abc", "123", ""} - service := s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL}) + s.config = &s3test.Config{ + Send409Conflict: true, + } + srv, err := s3test.NewServer(s.config) + OK(t, err) + s.srv = srv + + s.region = aws.Region{ + Name: "faux-region-1", + S3Endpoint: srv.URL(), + S3LocationConstraint: true, // s3test server requires a LocationConstraint + } + + s.auth = aws.Auth{"abc", "123", ""} + + service := s3.New(s.auth, s.region) bucket := service.Bucket("testbucket") - err := bucket.PutBucket("private") + err = bucket.PutBucket("private") OK(t, err) - t.Logf("created s3 backend locally at %s", testServer.URL) + t.Logf("created s3 backend locally") return bes3.OpenS3Bucket(bucket, "testbucket") } -func teardownS3Backend(t *testing.T, b *bes3.S3) { - if !*TestCleanup { - t.Logf("leaving backend at %s\n", b.Location()) - return - } - - testServer.Flush() -} - func TestS3Backend(t *testing.T) { s := setupS3Backend(t) - defer teardownS3Backend(t, s) testBackend(s, t) } From aaae7f33d33fd0b86c79e68543eda5a107ca141e Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 07:17:38 -0500 Subject: [PATCH 08/13] Have number of connections limited by channel Removes previous limit of 1 connection --- backend/s3/s3.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 991385525..4f12251cd 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -5,7 +5,6 @@ import ( "errors" "io" "strings" - "sync" "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/s3" @@ -14,6 +13,7 @@ import ( ) const maxKeysInList = 1000 +const connLimit = 10 func s3path(t backend.Type, name string) string { if t == backend.Config { @@ -23,18 +23,23 @@ func s3path(t backend.Type, name string) string { } type S3 struct { - bucket *s3.Bucket - mput sync.Mutex - path string + bucket *s3.Bucket + connChan chan struct{} + path string } // Open a backend using an S3 bucket object func OpenS3Bucket(bucket *s3.Bucket, bucketname string) *S3 { - return &S3{bucket: bucket, path: bucketname} + connChan := make(chan struct{}, connLimit) + for i := 0; i < connLimit; i++ { + connChan <- struct{}{} + } + + return &S3{bucket: bucket, path: bucketname, connChan: connChan} } // Open opens the s3 backend at bucket and region. -func Open(regionname, bucketname string) (*S3, error) { +func Open(regionname, bucketname string) (backend.Backend, error) { auth, err := aws.EnvAuth() if err != nil { return nil, err @@ -42,7 +47,7 @@ func Open(regionname, bucketname string) (*S3, error) { client := s3.New(auth, aws.Regions[regionname]) - return &S3{bucket: client.Bucket(bucketname), path: bucketname}, nil + return OpenS3Bucket(client.Bucket(bucketname), bucketname), nil } // Location returns this backend's location (the bucket name). @@ -94,9 +99,9 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { return errors.New("key already exists!") } - bb.b.mput.Lock() + <-bb.b.connChan err = bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") - bb.b.mput.Unlock() + bb.b.connChan <- struct{}{} bb.buf.Reset() return err } @@ -118,7 +123,9 @@ func (b *S3) get(t backend.Type, name string) (*s3Blob, error) { } path := s3path(t, name) + <-b.connChan data, err := b.bucket.Get(path) + b.connChan <- struct{}{} blob.buf = bytes.NewBuffer(data) return blob, err } From d30a95e8ad8bf2839b57c79a5d7cfa57e1227460 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 07:47:11 -0500 Subject: [PATCH 09/13] Return bucket readers instead of buffering a key on read. --- backend/s3/s3.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 4f12251cd..22267c3c1 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -3,7 +3,9 @@ package s3 import ( "bytes" "errors" + "fmt" "io" + "io/ioutil" "strings" "github.com/mitchellh/goamz/aws" @@ -117,40 +119,33 @@ func (b *S3) Create() (backend.Blob, error) { return &blob, nil } -func (b *S3) get(t backend.Type, name string) (*s3Blob, error) { - blob := &s3Blob{ - b: b, - } - - path := s3path(t, name) - <-b.connChan - data, err := b.bucket.Get(path) - b.connChan <- struct{}{} - blob.buf = bytes.NewBuffer(data) - return blob, err -} - // Get returns a reader that yields the content stored under the given // name. The reader should be closed after draining it. func (b *S3) Get(t backend.Type, name string) (io.ReadCloser, error) { - return b.get(t, name) + path := s3path(t, name) + return b.bucket.GetReader(path) } // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. func (b *S3) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - blob, err := b.get(t, name) + rc, err := b.Get(t, name) if err != nil { return nil, err } - blob.buf.Next(int(offset)) - - if length == 0 { - return blob, nil + n, errc := io.CopyN(ioutil.Discard, rc, int64(offset)) + if errc != nil { + return nil, errc + } else if n != int64(offset) { + return nil, fmt.Errorf("less bytes read than expected, read: %d, expected: %d", n, offset) } - return backend.LimitReadCloser(blob, int64(length)), nil + if length == 0 { + return rc, nil + } + + return backend.LimitReadCloser(rc, int64(length)), nil } // Test returns true if a blob of the given type and name exists in the backend. From e963faf846ec58d1a613819d7a42c28e5f0db5e6 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 08:02:29 -0500 Subject: [PATCH 10/13] Place all restic data under a key prefix. Deleting a restic repository now only deletes restic keys, leaving the rest of the bucket in tact. --- backend/s3/s3.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 22267c3c1..fd4ce88a2 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -16,12 +16,13 @@ import ( const maxKeysInList = 1000 const connLimit = 10 +const backendPrefix = "restic" func s3path(t backend.Type, name string) string { if t == backend.Config { - return string(t) + return backendPrefix + "/" + string(t) } - return string(t) + "/" + name + return backendPrefix + "/" + string(t) + "/" + name } type S3 struct { @@ -173,7 +174,7 @@ func (b *S3) Remove(t backend.Type, name string) error { func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { ch := make(chan string) - prefix := string(t) + "/" + prefix := s3path(t, "") listresp, err := b.bucket.List(prefix, "/", "", maxKeysInList) @@ -218,9 +219,24 @@ func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { return ch } -// Delete removes the repository and all files. +// Remove keys for a specified backend type +func (b *S3) removeKeys(t backend.Type) { + doneChan := make(chan struct{}) + for key := range b.List(backend.Data, doneChan) { + b.Remove(backend.Data, key) + } + doneChan <- struct{}{} +} + +// Delete removes all restic keys func (b *S3) Delete() error { - return b.bucket.DelBucket() + b.removeKeys(backend.Data) + b.removeKeys(backend.Key) + b.removeKeys(backend.Lock) + b.removeKeys(backend.Snapshot) + b.removeKeys(backend.Index) + b.removeKeys(backend.Config) + return nil } // Close does nothing From 57ee1db8aa53d222a7190ff52f5e67f78bc8777f Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 08:08:39 -0500 Subject: [PATCH 11/13] Rename backend type to S3Backend --- backend/s3/s3.go | 60 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index fd4ce88a2..d26b20bcd 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -25,20 +25,20 @@ func s3path(t backend.Type, name string) string { return backendPrefix + "/" + string(t) + "/" + name } -type S3 struct { +type S3Backend struct { bucket *s3.Bucket connChan chan struct{} path string } // Open a backend using an S3 bucket object -func OpenS3Bucket(bucket *s3.Bucket, bucketname string) *S3 { +func OpenS3Bucket(bucket *s3.Bucket, bucketname string) *S3Backend { connChan := make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { connChan <- struct{}{} } - return &S3{bucket: bucket, path: bucketname, connChan: connChan} + return &S3Backend{bucket: bucket, path: bucketname, connChan: connChan} } // Open opens the s3 backend at bucket and region. @@ -54,12 +54,12 @@ func Open(regionname, bucketname string) (backend.Backend, error) { } // Location returns this backend's location (the bucket name). -func (b *S3) Location() string { - return b.path +func (be *S3Backend) Location() string { + return be.path } type s3Blob struct { - b *S3 + b *S3Backend buf *bytes.Buffer final bool } @@ -111,9 +111,9 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { // Create creates a new Blob. The data is available only after Finalize() // has been called on the returned Blob. -func (b *S3) Create() (backend.Blob, error) { +func (be *S3Backend) Create() (backend.Blob, error) { blob := s3Blob{ - b: b, + b: be, buf: &bytes.Buffer{}, } @@ -122,15 +122,15 @@ func (b *S3) Create() (backend.Blob, error) { // Get returns a reader that yields the content stored under the given // name. The reader should be closed after draining it. -func (b *S3) Get(t backend.Type, name string) (io.ReadCloser, error) { +func (be *S3Backend) Get(t backend.Type, name string) (io.ReadCloser, error) { path := s3path(t, name) - return b.bucket.GetReader(path) + return be.bucket.GetReader(path) } // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. -func (b *S3) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - rc, err := b.Get(t, name) +func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { + rc, err := be.Get(t, name) if err != nil { return nil, err } @@ -150,10 +150,10 @@ func (b *S3) GetReader(t backend.Type, name string, offset, length uint) (io.Rea } // Test returns true if a blob of the given type and name exists in the backend. -func (b *S3) Test(t backend.Type, name string) (bool, error) { +func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { found := false path := s3path(t, name) - key, err := b.bucket.GetKey(path) + key, err := be.bucket.GetKey(path) if err == nil && key.Key == path { found = true } @@ -163,20 +163,20 @@ func (b *S3) Test(t backend.Type, name string) (bool, error) { } // Remove removes the blob with the given name and type. -func (b *S3) Remove(t backend.Type, name string) error { +func (be *S3Backend) Remove(t backend.Type, name string) error { path := s3path(t, name) - return b.bucket.Del(path) + return be.bucket.Del(path) } // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { +func (be *S3Backend) List(t backend.Type, done <-chan struct{}) <-chan string { ch := make(chan string) prefix := s3path(t, "") - listresp, err := b.bucket.List(prefix, "/", "", maxKeysInList) + listresp, err := be.bucket.List(prefix, "/", "", maxKeysInList) if err != nil { close(ch) @@ -190,7 +190,7 @@ func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { // Continue making requests to get full list. for listresp.IsTruncated { - listresp, err = b.bucket.List(prefix, "/", listresp.NextMarker, maxKeysInList) + listresp, err = be.bucket.List(prefix, "/", listresp.NextMarker, maxKeysInList) if err != nil { close(ch) return ch @@ -220,24 +220,24 @@ func (b *S3) List(t backend.Type, done <-chan struct{}) <-chan string { } // Remove keys for a specified backend type -func (b *S3) removeKeys(t backend.Type) { +func (be *S3Backend) removeKeys(t backend.Type) { doneChan := make(chan struct{}) - for key := range b.List(backend.Data, doneChan) { - b.Remove(backend.Data, key) + for key := range be.List(backend.Data, doneChan) { + be.Remove(backend.Data, key) } doneChan <- struct{}{} } // Delete removes all restic keys -func (b *S3) Delete() error { - b.removeKeys(backend.Data) - b.removeKeys(backend.Key) - b.removeKeys(backend.Lock) - b.removeKeys(backend.Snapshot) - b.removeKeys(backend.Index) - b.removeKeys(backend.Config) +func (be *S3Backend) Delete() error { + be.removeKeys(backend.Data) + be.removeKeys(backend.Key) + be.removeKeys(backend.Lock) + be.removeKeys(backend.Snapshot) + be.removeKeys(backend.Index) + be.removeKeys(backend.Config) return nil } // Close does nothing -func (b *S3) Close() error { return nil } +func (be *S3Backend) Close() error { return nil } From 12596c6841ce789b3381ced52ad2dd86859a5a88 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 08:43:47 -0500 Subject: [PATCH 12/13] Fix test to use new type name on backend --- backend/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/s3_test.go b/backend/s3_test.go index 64a23b9a1..1b021e92f 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -20,7 +20,7 @@ type LocalServer struct { var s LocalServer -func setupS3Backend(t *testing.T) *bes3.S3 { +func setupS3Backend(t *testing.T) *bes3.S3Backend { s.config = &s3test.Config{ Send409Conflict: true, } From cf0531a2b199cbdf8edbc913ed7d6e94817d4bb8 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 14 Jun 2015 09:05:20 -0500 Subject: [PATCH 13/13] Pass buffer instead of byte slice s3 library creates internal buffer pass in byte slice --- backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index d26b20bcd..4b72d8ca5 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -103,7 +103,7 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { } <-bb.b.connChan - err = bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") + err = bb.b.bucket.PutReader(path, bb.buf, int64(bb.buf.Len()), "binary/octet-stream", "private") bb.b.connChan <- struct{}{} bb.buf.Reset() return err