r.loginov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #832 from BrianBland/ng-storagedriver-tests

Browse source 
Updates storagedriver tests to better test directory trees
This commit is contained in:
Stephen Day 2014-12-09 21:01:40 -08:00
commit 043c81bea2
6 changed files with 217 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
circle.yml
View file

@ -21,7 +21,7 @@ test:
- test -z $(gofmt -s -l . | tee /dev/stderr)
- go vet ./...
- test -z $(golint ./... | tee /dev/stderr)
- go test -test.v ./...
- go test -test.v -test.short ./...
# Disabling the race detector due to massive memory usage.
# - go test -race -test.v ./...:

4
storagedriver/filesystem/driver.go
View file

@ -203,6 +203,10 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
return storagedriver.PathNotFoundError{Path: sourcePath}
}
if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
return err
}
err := os.Rename(source, dest)
return err
}

20
storagedriver/inmemory/driver.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"io"
"io/ioutil"
"strings"
"sync"
"time"
@ -87,7 +86,7 @@ func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
path = d.normalize(path)
path = normalize(path)
found := d.root.find(path)
if found.path() != path {
@ -111,7 +110,7 @@ func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (nn in
return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
normalized := d.normalize(path)
normalized := normalize(path)
f, err := d.root.mkfile(normalized)
if err != nil {
@ -139,7 +138,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
normalized := d.normalize(path)
normalized := normalize(path)
found := d.root.find(path)
if found.path() != normalized {
@ -162,7 +161,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
// List returns a list of the objects that are direct descendants of the given
// path.
func (d *Driver) List(path string) ([]string, error) {
normalized := d.normalize(path)
normalized := normalize(path)
found := d.root.find(normalized)
@ -192,7 +191,7 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
d.mutex.Lock()
defer d.mutex.Unlock()
normalizedSrc, normalizedDst := d.normalize(sourcePath), d.normalize(destPath)
normalizedSrc, normalizedDst := normalize(sourcePath), normalize(destPath)
err := d.root.move(normalizedSrc, normalizedDst)
switch err {
@ -208,7 +207,7 @@ func (d *Driver) Delete(path string) error {
d.mutex.Lock()
defer d.mutex.Unlock()
normalized := d.normalize(path)
normalized := normalize(path)
err := d.root.delete(normalized)
switch err {
@ -218,10 +217,3 @@ func (d *Driver) Delete(path string) error {
return err
}
}
func (d *Driver) normalize(p string) string {
if !strings.HasPrefix(p, "/") {
p = "/" + p // Ghetto path absolution.
}
return p
}

14
storagedriver/inmemory/mfs.go
View file

@ -11,7 +11,7 @@ import (
var (
errExists = fmt.Errorf("exists")
errNotExists = fmt.Errorf("exists")
errNotExists = fmt.Errorf("notexists")
errIsNotDir = fmt.Errorf("notdir")
errIsDir = fmt.Errorf("isdir")
)
@ -139,9 +139,7 @@ func (d *dir) mkfile(p string) (*file, error) {
// mkdirs creates any missing directory entries in p and returns the result.
func (d *dir) mkdirs(p string) (*dir, error) {
if p == "" {
p = "/"
}
p = normalize(p)
n := d.find(p)
@ -210,7 +208,7 @@ func (d *dir) move(src, dst string) error {
srcDirname, srcFilename := path.Split(src)
sp := d.find(srcDirname)
if sp.path() != srcDirname {
if normalize(srcDirname) != normalize(sp.path()) {
return errNotExists
}
@ -237,7 +235,7 @@ func (d *dir) delete(p string) error {
dirname, filename := path.Split(p)
parent := d.find(dirname)
if dirname != parent.path() {
if normalize(dirname) != normalize(parent.path()) {
return errNotExists
}
@ -328,3 +326,7 @@ func (c *common) path() string {
func (c *common) modtime() time.Time {
return c.mod
}
func normalize(p string) string {
return "/" + strings.Trim(p, "/")
}

31
storagedriver/s3/s3_test.go
View file

@ -20,32 +20,37 @@ func Test(t *testing.T) { check.TestingT(t) }
func init() {
accessKey := os.Getenv("AWS_ACCESS_KEY")
secretKey := os.Getenv("AWS_SECRET_KEY")
region := os.Getenv("AWS_REGION")
bucket := os.Getenv("S3_BUCKET")
encrypt := os.Getenv("S3_ENCRYPT")
s3DriverConstructor := func() (storagedriver.StorageDriver, error) {
s3DriverConstructor := func(region aws.Region) (storagedriver.StorageDriver, error) {
shouldEncrypt, err := strconv.ParseBool(encrypt)
if err != nil {
return nil, err
}
return New(accessKey, secretKey, aws.GetRegion(region), shouldEncrypt, bucket)
return New(accessKey, secretKey, region, shouldEncrypt, bucket)
}
// Skip S3 storage driver tests if environment variable parameters are not provided
skipCheck := func() string {
if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
if accessKey == "" || secretKey == "" || bucket == "" || encrypt == "" {
return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
}
return ""
}
testsuites.RegisterInProcessSuite(s3DriverConstructor, skipCheck)
testsuites.RegisterIPCSuite(driverName, map[string]string{
"accesskey": accessKey,
"secretkey": secretKey,
"region": region,
"bucket": bucket,
"encrypt": encrypt,
}, skipCheck)
for _, region := range aws.Regions {
if region == aws.USGovWest {
continue
}
testsuites.RegisterInProcessSuite(s3DriverConstructor(region), skipCheck)
testsuites.RegisterIPCSuite(driverName, map[string]string{
"accesskey": accessKey,
"secretkey": secretKey,
"region": region.Name,
"bucket": bucket,
"encrypt": encrypt,
}, skipCheck)
}
}

237
storagedriver/testsuites/testsuites.go
View file

@ -2,6 +2,7 @@ package testsuites
import (
"bytes"
"crypto/sha1"
"io"
"io/ioutil"
"math/rand"
@ -109,35 +110,54 @@ func (suite *DriverSuite) TearDownSuite(c *check.C) {
// TestWriteRead1 tests a simple write-read workflow.
func (suite *DriverSuite) TestWriteRead1(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("a")
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead2 tests a simple write-read workflow with unicode data.
func (suite *DriverSuite) TestWriteRead2(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("\xc3\x9f")
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead3 tests a simple write-read workflow with a small string.
func (suite *DriverSuite) TestWriteRead3(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead4 tests a simple write-read workflow with 1MB of data.
func (suite *DriverSuite) TestWriteRead4(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(1024 * 1024))
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompare(c, filename, contents)
}
// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage
// driver safely.
func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) {
filename := randomPath(32)
contents := []byte{0x80, 0x80, 0x80, 0x80}
suite.writeReadCompare(c, filename, contents)
}
// TestTruncate tests that putting smaller contents than an original file does
// remove the excess contents.
func (suite *DriverSuite) TestTruncate(c *check.C) {
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompare(c, filename, contents)
contents = randomContents(1024)
suite.writeReadCompare(c, filename, contents)
}
// TestReadNonexistent tests reading content from an empty path.
func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
_, err := suite.StorageDriver.GetContent(filename)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -145,7 +165,7 @@ func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
// TestWriteReadStreams1 tests a simple write-read streaming workflow.
func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("a")
suite.writeReadCompareStreams(c, filename, contents)
}
@ -153,7 +173,7 @@ func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
// TestWriteReadStreams2 tests a simple write-read streaming workflow with
// unicode data.
func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("\xc3\x9f")
suite.writeReadCompareStreams(c, filename, contents)
}
@ -161,30 +181,68 @@ func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
// TestWriteReadStreams3 tests a simple write-read streaming workflow with a
// small amount of data.
func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB
// of data.
func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(1024 * 1024))
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the
// storage driver safely.
func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) {
filename := randomPath(32)
contents := []byte{0x80, 0x80, 0x80, 0x80}
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage
// driver safely.
func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) {
if testing.Short() {
c.Skip("Skipping test in short mode")
}
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
checksum := sha1.New()
var offset int64
var chunkSize int64 = 1024 * 1024
for i := 0; i < 5*1024; i++ {
contents := randomContents(chunkSize)
written, err := suite.StorageDriver.WriteStream(filename, offset, io.TeeReader(bytes.NewReader(contents), checksum))
c.Assert(err, check.IsNil)
c.Assert(written, check.Equals, chunkSize)
offset += chunkSize
}
reader, err := suite.StorageDriver.ReadStream(filename, 0)
c.Assert(err, check.IsNil)
writtenChecksum := sha1.New()
io.Copy(writtenChecksum, reader)
c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil))
}
// TestReadStreamWithOffset tests that the appropriate data is streamed when
// reading with a given offset.
func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) {
filename := randomString(32)
defer suite.StorageDriver.Delete(filename)
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
chunkSize := int64(32)
contentsChunk1 := []byte(randomString(chunkSize))
contentsChunk2 := []byte(randomString(chunkSize))
contentsChunk3 := []byte(randomString(chunkSize))
contentsChunk1 := randomContents(chunkSize)
contentsChunk2 := randomContents(chunkSize)
contentsChunk3 := randomContents(chunkSize)
err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
c.Assert(err, check.IsNil)
@ -256,15 +314,15 @@ func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) {
// TestContinueStreamAppend tests that a stream write can be appended to without
// corrupting the data.
func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) {
filename := randomString(32)
defer suite.StorageDriver.Delete(filename)
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
chunkSize := int64(10 * 1024 * 1024)
contentsChunk1 := []byte(randomString(chunkSize))
contentsChunk2 := []byte(randomString(chunkSize))
contentsChunk3 := []byte(randomString(chunkSize))
contentsChunk4 := []byte(randomString(chunkSize))
contentsChunk1 := randomContents(chunkSize)
contentsChunk2 := randomContents(chunkSize)
contentsChunk3 := randomContents(chunkSize)
contentsChunk4 := randomContents(chunkSize)
zeroChunk := make([]byte, int64(chunkSize))
fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)
@ -337,7 +395,7 @@ func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) {
// TestReadNonexistentStream tests that reading a stream for a nonexistent path
// fails.
func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
_, err := suite.StorageDriver.ReadStream(filename, 0)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -345,15 +403,15 @@ func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
// TestList checks the returned list of keys after populating a directory tree.
func (suite *DriverSuite) TestList(c *check.C) {
rootDirectory := "/" + randomString(int64(8+rand.Intn(8)))
defer suite.StorageDriver.Delete(rootDirectory)
rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8)))
defer suite.StorageDriver.Delete("/")
parentDirectory := rootDirectory + "/" + randomString(int64(8+rand.Intn(8)))
parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
childFiles := make([]string, 50)
for i := 0; i < len(childFiles); i++ {
childFile := parentDirectory + "/" + randomString(int64(8+rand.Intn(8)))
childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
childFiles[i] = childFile
err := suite.StorageDriver.PutContent(childFile, []byte(randomString(32)))
err := suite.StorageDriver.PutContent(childFile, randomContents(32))
c.Assert(err, check.IsNil)
}
sort.Strings(childFiles)
@ -381,12 +439,12 @@ func (suite *DriverSuite) TestList(c *check.C) {
// TestMove checks that a moved object no longer exists at the source path and
// does exist at the destination.
func (suite *DriverSuite) TestMove(c *check.C) {
contents := []byte(randomString(32))
sourcePath := randomString(32)
destPath := randomString(32)
contents := randomContents(32)
sourcePath := randomPath(32)
destPath := randomPath(32)
defer suite.StorageDriver.Delete(sourcePath)
defer suite.StorageDriver.Delete(destPath)
defer suite.StorageDriver.Delete(firstPart(sourcePath))
defer suite.StorageDriver.Delete(firstPart(destPath))
err := suite.StorageDriver.PutContent(sourcePath, contents)
c.Assert(err, check.IsNil)
@ -405,8 +463,8 @@ func (suite *DriverSuite) TestMove(c *check.C) {
// TestMoveNonexistent checks that moving a nonexistent key fails
func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
sourcePath := randomString(32)
destPath := randomString(32)
sourcePath := randomPath(32)
destPath := randomPath(32)
err := suite.StorageDriver.Move(sourcePath, destPath)
c.Assert(err, check.NotNil)
@ -416,10 +474,10 @@ func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
// TestDelete checks that the delete operation removes data from the storage
// driver
func (suite *DriverSuite) TestDelete(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
err := suite.StorageDriver.PutContent(filename, contents)
c.Assert(err, check.IsNil)
@ -434,7 +492,7 @@ func (suite *DriverSuite) TestDelete(c *check.C) {
// TestDeleteNonexistent checks that removing a nonexistent key fails.
func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
err := suite.StorageDriver.Delete(filename)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -442,13 +500,13 @@ func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
// TestDeleteFolder checks that deleting a folder removes all child elements.
func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
dirname := randomString(32)
filename1 := randomString(32)
filename2 := randomString(32)
contents := []byte(randomString(32))
dirname := randomPath(32)
filename1 := randomPath(32)
filename2 := randomPath(32)
filename3 := randomPath(32)
contents := randomContents(32)
defer suite.StorageDriver.Delete(path.Join(dirname, filename1))
defer suite.StorageDriver.Delete(path.Join(dirname, filename2))
defer suite.StorageDriver.Delete(firstPart(dirname))
err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents)
c.Assert(err, check.IsNil)
@ -456,6 +514,22 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents)
c.Assert(err, check.IsNil)
err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents)
c.Assert(err, check.IsNil)
err = suite.StorageDriver.Delete(path.Join(dirname, filename1))
c.Assert(err, check.IsNil)
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2))
c.Assert(err, check.IsNil)
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3))
c.Assert(err, check.IsNil)
err = suite.StorageDriver.Delete(dirname)
c.Assert(err, check.IsNil)
@ -466,22 +540,28 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
}
// TestStatCall runs verifies the implementation of the storagedriver's Stat call.
func (suite *DriverSuite) TestStatCall(c *check.C) {
content := randomString(4096)
dirPath := randomString(32)
fileName := randomString(32)
content := randomContents(4096)
dirPath := randomPath(32)
fileName := randomFilename(32)
filePath := path.Join(dirPath, fileName)
defer suite.StorageDriver.Delete(dirPath)
// Call on non-existent file/dir, check error.
fi, err := suite.StorageDriver.Stat(filePath)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
c.Assert(fi, check.IsNil)
err = suite.StorageDriver.PutContent(filePath, []byte(content))
err = suite.StorageDriver.PutContent(filePath, content)
c.Assert(err, check.IsNil)
// Call on regular file, check results
@ -555,7 +635,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
tfName := path.Base(tf.Name())
defer suite.StorageDriver.Delete(tfName)
contents := []byte(randomString(size))
contents := randomContents(size)
_, err = tf.Write(contents)
c.Assert(err, check.IsNil)
@ -578,7 +658,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
}
func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) {
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
err := suite.StorageDriver.PutContent(filename, contents)
c.Assert(err, check.IsNil)
@ -590,7 +670,7 @@ func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents
}
func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) {
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents))
c.Assert(err, check.IsNil)
@ -606,12 +686,55 @@ func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, c
c.Assert(readContents, check.DeepEquals, contents)
}
var pathChars = []byte("abcdefghijklmnopqrstuvwxyz")
var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
func randomString(length int64) string {
func randomPath(length int64) string {
path := ""
for int64(len(path)) < length {
chunkLength := rand.Int63n(length-int64(len(path))) + 1
chunk := randomFilename(chunkLength)
path += chunk
if length-int64(len(path)) == 1 {
path += randomFilename(1)
} else if length-int64(len(path)) > 1 {
path += "/"
}
}
return path
}
func randomFilename(length int64) string {
b := make([]byte, length)
for i := range b {
b[i] = pathChars[rand.Intn(len(pathChars))]
b[i] = filenameChars[rand.Intn(len(filenameChars))]
}
return string(b)
}
func randomContents(length int64) []byte {
b := make([]byte, length)
for i := range b {
b[i] = byte(rand.Intn(2 << 8))
}
return b
}
func firstPart(filePath string) string {
for {
if filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
dir, file := path.Split(filePath)
if dir == "" && file == "" {
return "/"
}
if dir == "" {
return file
}
if file == "" {
return dir
}
filePath = dir
}
}