TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Uses random paths with slashes for storagedriver tests, adds edge cases

Browse source 
This replaces only using flat filenames, to better test nested file
behaviors.
Fixed inmemory/mfs.go and filesystem/driver.go after finding bugs with
the new tests and test behavior.
This commit is contained in:
Brian Bland 2014-12-08 18:22:08 -08:00
parent 9e38ca2231
commit cacf33ab62
3 changed files with 186 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
storagedriver/filesystem/driver.go
View file

@ -203,6 +203,10 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
return storagedriver.PathNotFoundError{Path: sourcePath}
}
if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
return err
}
err := os.Rename(source, dest)
return err
}

4
storagedriver/inmemory/mfs.go
View file

@ -210,7 +210,7 @@ func (d *dir) move(src, dst string) error {
srcDirname, srcFilename := path.Split(src)
sp := d.find(srcDirname)
if sp.path() != srcDirname {
if srcDirname != strings.TrimSuffix(sp.path(), "/")+"/" {
return errNotExists
}
@ -237,7 +237,7 @@ func (d *dir) delete(p string) error {
dirname, filename := path.Split(p)
parent := d.find(dirname)
if dirname != parent.path() {
if dirname != strings.TrimSuffix(parent.path(), "/")+"/" {
return errNotExists
}

237
storagedriver/testsuites/testsuites.go
View file

@ -2,6 +2,7 @@ package testsuites
import (
"bytes"
"crypto/sha1"
"io"
"io/ioutil"
"math/rand"
@ -109,35 +110,54 @@ func (suite *DriverSuite) TearDownSuite(c *check.C) {
// TestWriteRead1 tests a simple write-read workflow.
func (suite *DriverSuite) TestWriteRead1(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("a")
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead2 tests a simple write-read workflow with unicode data.
func (suite *DriverSuite) TestWriteRead2(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("\xc3\x9f")
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead3 tests a simple write-read workflow with a small string.
func (suite *DriverSuite) TestWriteRead3(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
suite.writeReadCompare(c, filename, contents)
}
// TestWriteRead4 tests a simple write-read workflow with 1MB of data.
func (suite *DriverSuite) TestWriteRead4(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(1024 * 1024))
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompare(c, filename, contents)
}
// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage
// driver safely.
func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) {
filename := randomPath(32)
contents := []byte{0x80, 0x80, 0x80, 0x80}
suite.writeReadCompare(c, filename, contents)
}
// TestTruncate tests that putting smaller contents than an original file does
// remove the excess contents.
func (suite *DriverSuite) TestTruncate(c *check.C) {
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompare(c, filename, contents)
contents = randomContents(1024)
suite.writeReadCompare(c, filename, contents)
}
// TestReadNonexistent tests reading content from an empty path.
func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
_, err := suite.StorageDriver.GetContent(filename)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -145,7 +165,7 @@ func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
// TestWriteReadStreams1 tests a simple write-read streaming workflow.
func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("a")
suite.writeReadCompareStreams(c, filename, contents)
}
@ -153,7 +173,7 @@ func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
// TestWriteReadStreams2 tests a simple write-read streaming workflow with
// unicode data.
func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
contents := []byte("\xc3\x9f")
suite.writeReadCompareStreams(c, filename, contents)
}
@ -161,30 +181,68 @@ func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
// TestWriteReadStreams3 tests a simple write-read streaming workflow with a
// small amount of data.
func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB
// of data.
func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(1024 * 1024))
filename := randomPath(32)
contents := randomContents(1024 * 1024)
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the
// storage driver safely.
func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) {
filename := randomPath(32)
contents := []byte{0x80, 0x80, 0x80, 0x80}
suite.writeReadCompareStreams(c, filename, contents)
}
// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage
// driver safely.
func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) {
if testing.Short() {
c.Skip("Skipping test in short mode")
}
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
checksum := sha1.New()
var offset int64 = 0
var chunkSize int64 = 1024 * 1024
for i := 0; i < 5*1024; i++ {
contents := randomContents(chunkSize)
written, err := suite.StorageDriver.WriteStream(filename, offset, io.TeeReader(bytes.NewReader(contents), checksum))
c.Assert(err, check.IsNil)
c.Assert(written, check.Equals, chunkSize)
offset += chunkSize
}
reader, err := suite.StorageDriver.ReadStream(filename, 0)
c.Assert(err, check.IsNil)
writtenChecksum := sha1.New()
io.Copy(writtenChecksum, reader)
c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil))
}
// TestReadStreamWithOffset tests that the appropriate data is streamed when
// reading with a given offset.
func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) {
filename := randomString(32)
defer suite.StorageDriver.Delete(filename)
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
chunkSize := int64(32)
contentsChunk1 := []byte(randomString(chunkSize))
contentsChunk2 := []byte(randomString(chunkSize))
contentsChunk3 := []byte(randomString(chunkSize))
contentsChunk1 := randomContents(chunkSize)
contentsChunk2 := randomContents(chunkSize)
contentsChunk3 := randomContents(chunkSize)
err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
c.Assert(err, check.IsNil)
@ -256,15 +314,15 @@ func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) {
// TestContinueStreamAppend tests that a stream write can be appended to without
// corrupting the data.
func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) {
filename := randomString(32)
defer suite.StorageDriver.Delete(filename)
filename := randomPath(32)
defer suite.StorageDriver.Delete(firstPart(filename))
chunkSize := int64(10 * 1024 * 1024)
contentsChunk1 := []byte(randomString(chunkSize))
contentsChunk2 := []byte(randomString(chunkSize))
contentsChunk3 := []byte(randomString(chunkSize))
contentsChunk4 := []byte(randomString(chunkSize))
contentsChunk1 := randomContents(chunkSize)
contentsChunk2 := randomContents(chunkSize)
contentsChunk3 := randomContents(chunkSize)
contentsChunk4 := randomContents(chunkSize)
zeroChunk := make([]byte, int64(chunkSize))
fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)
@ -337,7 +395,7 @@ func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) {
// TestReadNonexistentStream tests that reading a stream for a nonexistent path
// fails.
func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
_, err := suite.StorageDriver.ReadStream(filename, 0)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -345,15 +403,15 @@ func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
// TestList checks the returned list of keys after populating a directory tree.
func (suite *DriverSuite) TestList(c *check.C) {
rootDirectory := "/" + randomString(int64(8+rand.Intn(8)))
defer suite.StorageDriver.Delete(rootDirectory)
rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8)))
defer suite.StorageDriver.Delete("/")
parentDirectory := rootDirectory + "/" + randomString(int64(8+rand.Intn(8)))
parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
childFiles := make([]string, 50)
for i := 0; i < len(childFiles); i++ {
childFile := parentDirectory + "/" + randomString(int64(8+rand.Intn(8)))
childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
childFiles[i] = childFile
err := suite.StorageDriver.PutContent(childFile, []byte(randomString(32)))
err := suite.StorageDriver.PutContent(childFile, randomContents(32))
c.Assert(err, check.IsNil)
}
sort.Strings(childFiles)
@ -381,12 +439,12 @@ func (suite *DriverSuite) TestList(c *check.C) {
// TestMove checks that a moved object no longer exists at the source path and
// does exist at the destination.
func (suite *DriverSuite) TestMove(c *check.C) {
contents := []byte(randomString(32))
sourcePath := randomString(32)
destPath := randomString(32)
contents := randomContents(32)
sourcePath := randomPath(32)
destPath := randomPath(32)
defer suite.StorageDriver.Delete(sourcePath)
defer suite.StorageDriver.Delete(destPath)
defer suite.StorageDriver.Delete(firstPart(sourcePath))
defer suite.StorageDriver.Delete(firstPart(destPath))
err := suite.StorageDriver.PutContent(sourcePath, contents)
c.Assert(err, check.IsNil)
@ -405,8 +463,8 @@ func (suite *DriverSuite) TestMove(c *check.C) {
// TestMoveNonexistent checks that moving a nonexistent key fails
func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
sourcePath := randomString(32)
destPath := randomString(32)
sourcePath := randomPath(32)
destPath := randomPath(32)
err := suite.StorageDriver.Move(sourcePath, destPath)
c.Assert(err, check.NotNil)
@ -416,10 +474,10 @@ func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
// TestDelete checks that the delete operation removes data from the storage
// driver
func (suite *DriverSuite) TestDelete(c *check.C) {
filename := randomString(32)
contents := []byte(randomString(32))
filename := randomPath(32)
contents := randomContents(32)
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
err := suite.StorageDriver.PutContent(filename, contents)
c.Assert(err, check.IsNil)
@ -434,7 +492,7 @@ func (suite *DriverSuite) TestDelete(c *check.C) {
// TestDeleteNonexistent checks that removing a nonexistent key fails.
func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
filename := randomString(32)
filename := randomPath(32)
err := suite.StorageDriver.Delete(filename)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
@ -442,13 +500,13 @@ func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
// TestDeleteFolder checks that deleting a folder removes all child elements.
func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
dirname := randomString(32)
filename1 := randomString(32)
filename2 := randomString(32)
contents := []byte(randomString(32))
dirname := randomPath(32)
filename1 := randomPath(32)
filename2 := randomPath(32)
filename3 := randomPath(32)
contents := randomContents(32)
defer suite.StorageDriver.Delete(path.Join(dirname, filename1))
defer suite.StorageDriver.Delete(path.Join(dirname, filename2))
defer suite.StorageDriver.Delete(firstPart(dirname))
err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents)
c.Assert(err, check.IsNil)
@ -456,6 +514,22 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents)
c.Assert(err, check.IsNil)
err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents)
c.Assert(err, check.IsNil)
err = suite.StorageDriver.Delete(path.Join(dirname, filename1))
c.Assert(err, check.IsNil)
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2))
c.Assert(err, check.IsNil)
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3))
c.Assert(err, check.IsNil)
err = suite.StorageDriver.Delete(dirname)
c.Assert(err, check.IsNil)
@ -466,22 +540,28 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
_, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3))
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
}
// TestStatCall runs verifies the implementation of the storagedriver's Stat call.
func (suite *DriverSuite) TestStatCall(c *check.C) {
content := randomString(4096)
dirPath := randomString(32)
fileName := randomString(32)
content := randomContents(4096)
dirPath := randomPath(32)
fileName := randomFilename(32)
filePath := path.Join(dirPath, fileName)
defer suite.StorageDriver.Delete(dirPath)
// Call on non-existent file/dir, check error.
fi, err := suite.StorageDriver.Stat(filePath)
c.Assert(err, check.NotNil)
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
c.Assert(fi, check.IsNil)
err = suite.StorageDriver.PutContent(filePath, []byte(content))
err = suite.StorageDriver.PutContent(filePath, content)
c.Assert(err, check.IsNil)
// Call on regular file, check results
@ -555,7 +635,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
tfName := path.Base(tf.Name())
defer suite.StorageDriver.Delete(tfName)
contents := []byte(randomString(size))
contents := randomContents(size)
_, err = tf.Write(contents)
c.Assert(err, check.IsNil)
@ -578,7 +658,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
}
func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) {
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
err := suite.StorageDriver.PutContent(filename, contents)
c.Assert(err, check.IsNil)
@ -590,7 +670,7 @@ func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents
}
func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) {
defer suite.StorageDriver.Delete(filename)
defer suite.StorageDriver.Delete(firstPart(filename))
nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents))
c.Assert(err, check.IsNil)
@ -606,12 +686,55 @@ func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, c
c.Assert(readContents, check.DeepEquals, contents)
}
var pathChars = []byte("abcdefghijklmnopqrstuvwxyz")
var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
func randomString(length int64) string {
func randomPath(length int64) string {
path := ""
for int64(len(path)) < length {
chunkLength := rand.Int63n(length-int64(len(path))) + 1
chunk := randomFilename(chunkLength)
path += chunk
if length-int64(len(path)) == 1 {
path += randomFilename(1)
} else if length-int64(len(path)) > 1 {
path += "/"
}
}
return path
}
func randomFilename(length int64) string {
b := make([]byte, length)
for i := range b {
b[i] = pathChars[rand.Intn(len(pathChars))]
b[i] = filenameChars[rand.Intn(len(filenameChars))]
}
return string(b)
}
func randomContents(length int64) []byte {
b := make([]byte, length)
for i := range b {
b[i] = byte(rand.Intn(2 << 8))
}
return b
}
func firstPart(filePath string) string {
for {
if filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
dir, file := path.Split(filePath)
if dir == "" && file == "" {
return "/"
}
if dir == "" {
return file
}
if file == "" {
return dir
}
filePath = dir
}
}