diff --git a/storagedriver/filesystem/driver.go b/storagedriver/filesystem/driver.go index 05ec6175b..3e352125f 100644 --- a/storagedriver/filesystem/driver.go +++ b/storagedriver/filesystem/driver.go @@ -203,6 +203,10 @@ func (d *Driver) Move(sourcePath string, destPath string) error { return storagedriver.PathNotFoundError{Path: sourcePath} } + if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { + return err + } + err := os.Rename(source, dest) return err } diff --git a/storagedriver/inmemory/mfs.go b/storagedriver/inmemory/mfs.go index 5248bbc6c..9eeac0da4 100644 --- a/storagedriver/inmemory/mfs.go +++ b/storagedriver/inmemory/mfs.go @@ -210,7 +210,7 @@ func (d *dir) move(src, dst string) error { srcDirname, srcFilename := path.Split(src) sp := d.find(srcDirname) - if sp.path() != srcDirname { + if srcDirname != strings.TrimSuffix(sp.path(), "/")+"/" { return errNotExists } @@ -237,7 +237,7 @@ func (d *dir) delete(p string) error { dirname, filename := path.Split(p) parent := d.find(dirname) - if dirname != parent.path() { + if dirname != strings.TrimSuffix(parent.path(), "/")+"/" { return errNotExists } diff --git a/storagedriver/testsuites/testsuites.go b/storagedriver/testsuites/testsuites.go index f745781e7..1abda7157 100644 --- a/storagedriver/testsuites/testsuites.go +++ b/storagedriver/testsuites/testsuites.go @@ -2,6 +2,7 @@ package testsuites import ( "bytes" + "crypto/sha1" "io" "io/ioutil" "math/rand" @@ -109,35 +110,54 @@ func (suite *DriverSuite) TearDownSuite(c *check.C) { // TestWriteRead1 tests a simple write-read workflow. func (suite *DriverSuite) TestWriteRead1(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("a") suite.writeReadCompare(c, filename, contents) } // TestWriteRead2 tests a simple write-read workflow with unicode data. func (suite *DriverSuite) TestWriteRead2(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompare(c, filename, contents) } // TestWriteRead3 tests a simple write-read workflow with a small string. func (suite *DriverSuite) TestWriteRead3(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) suite.writeReadCompare(c, filename, contents) } // TestWriteRead4 tests a simple write-read workflow with 1MB of data. func (suite *DriverSuite) TestWriteRead4(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(1024 * 1024)) + filename := randomPath(32) + contents := randomContents(1024 * 1024) + suite.writeReadCompare(c, filename, contents) +} + +// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage +// driver safely. +func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) { + filename := randomPath(32) + contents := []byte{0x80, 0x80, 0x80, 0x80} + suite.writeReadCompare(c, filename, contents) +} + +// TestTruncate tests that putting smaller contents than an original file does +// remove the excess contents. +func (suite *DriverSuite) TestTruncate(c *check.C) { + filename := randomPath(32) + contents := randomContents(1024 * 1024) + suite.writeReadCompare(c, filename, contents) + + contents = randomContents(1024) suite.writeReadCompare(c, filename, contents) } // TestReadNonexistent tests reading content from an empty path. func (suite *DriverSuite) TestReadNonexistent(c *check.C) { - filename := randomString(32) + filename := randomPath(32) _, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -145,7 +165,7 @@ func (suite *DriverSuite) TestReadNonexistent(c *check.C) { // TestWriteReadStreams1 tests a simple write-read streaming workflow. func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("a") suite.writeReadCompareStreams(c, filename, contents) } @@ -153,7 +173,7 @@ func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { // TestWriteReadStreams2 tests a simple write-read streaming workflow with // unicode data. func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { - filename := randomString(32) + filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompareStreams(c, filename, contents) } @@ -161,30 +181,68 @@ func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { // TestWriteReadStreams3 tests a simple write-read streaming workflow with a // small amount of data. func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB // of data. func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(1024 * 1024)) + filename := randomPath(32) + contents := randomContents(1024 * 1024) suite.writeReadCompareStreams(c, filename, contents) } +// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the +// storage driver safely. +func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) { + filename := randomPath(32) + contents := []byte{0x80, 0x80, 0x80, 0x80} + suite.writeReadCompareStreams(c, filename, contents) +} + +// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage +// driver safely. +func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) { + if testing.Short() { + c.Skip("Skipping test in short mode") + } + + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) + + checksum := sha1.New() + var offset int64 = 0 + var chunkSize int64 = 1024 * 1024 + + for i := 0; i < 5*1024; i++ { + contents := randomContents(chunkSize) + written, err := suite.StorageDriver.WriteStream(filename, offset, io.TeeReader(bytes.NewReader(contents), checksum)) + c.Assert(err, check.IsNil) + c.Assert(written, check.Equals, chunkSize) + offset += chunkSize + } + reader, err := suite.StorageDriver.ReadStream(filename, 0) + c.Assert(err, check.IsNil) + + writtenChecksum := sha1.New() + io.Copy(writtenChecksum, reader) + + c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil)) +} + // TestReadStreamWithOffset tests that the appropriate data is streamed when // reading with a given offset. func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { - filename := randomString(32) - defer suite.StorageDriver.Delete(filename) + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) chunkSize := int64(32) - contentsChunk1 := []byte(randomString(chunkSize)) - contentsChunk2 := []byte(randomString(chunkSize)) - contentsChunk3 := []byte(randomString(chunkSize)) + contentsChunk1 := randomContents(chunkSize) + contentsChunk2 := randomContents(chunkSize) + contentsChunk3 := randomContents(chunkSize) err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) c.Assert(err, check.IsNil) @@ -256,15 +314,15 @@ func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { // TestContinueStreamAppend tests that a stream write can be appended to without // corrupting the data. func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) { - filename := randomString(32) - defer suite.StorageDriver.Delete(filename) + filename := randomPath(32) + defer suite.StorageDriver.Delete(firstPart(filename)) chunkSize := int64(10 * 1024 * 1024) - contentsChunk1 := []byte(randomString(chunkSize)) - contentsChunk2 := []byte(randomString(chunkSize)) - contentsChunk3 := []byte(randomString(chunkSize)) - contentsChunk4 := []byte(randomString(chunkSize)) + contentsChunk1 := randomContents(chunkSize) + contentsChunk2 := randomContents(chunkSize) + contentsChunk3 := randomContents(chunkSize) + contentsChunk4 := randomContents(chunkSize) zeroChunk := make([]byte, int64(chunkSize)) fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...) @@ -337,7 +395,7 @@ func (suite *DriverSuite) TestContinueStreamAppend(c *check.C) { // TestReadNonexistentStream tests that reading a stream for a nonexistent path // fails. func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { - filename := randomString(32) + filename := randomPath(32) _, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -345,15 +403,15 @@ func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { // TestList checks the returned list of keys after populating a directory tree. func (suite *DriverSuite) TestList(c *check.C) { - rootDirectory := "/" + randomString(int64(8+rand.Intn(8))) - defer suite.StorageDriver.Delete(rootDirectory) + rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8))) + defer suite.StorageDriver.Delete("/") - parentDirectory := rootDirectory + "/" + randomString(int64(8+rand.Intn(8))) + parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles := make([]string, 50) for i := 0; i < len(childFiles); i++ { - childFile := parentDirectory + "/" + randomString(int64(8+rand.Intn(8))) + childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles[i] = childFile - err := suite.StorageDriver.PutContent(childFile, []byte(randomString(32))) + err := suite.StorageDriver.PutContent(childFile, randomContents(32)) c.Assert(err, check.IsNil) } sort.Strings(childFiles) @@ -381,12 +439,12 @@ func (suite *DriverSuite) TestList(c *check.C) { // TestMove checks that a moved object no longer exists at the source path and // does exist at the destination. func (suite *DriverSuite) TestMove(c *check.C) { - contents := []byte(randomString(32)) - sourcePath := randomString(32) - destPath := randomString(32) + contents := randomContents(32) + sourcePath := randomPath(32) + destPath := randomPath(32) - defer suite.StorageDriver.Delete(sourcePath) - defer suite.StorageDriver.Delete(destPath) + defer suite.StorageDriver.Delete(firstPart(sourcePath)) + defer suite.StorageDriver.Delete(firstPart(destPath)) err := suite.StorageDriver.PutContent(sourcePath, contents) c.Assert(err, check.IsNil) @@ -405,8 +463,8 @@ func (suite *DriverSuite) TestMove(c *check.C) { // TestMoveNonexistent checks that moving a nonexistent key fails func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { - sourcePath := randomString(32) - destPath := randomString(32) + sourcePath := randomPath(32) + destPath := randomPath(32) err := suite.StorageDriver.Move(sourcePath, destPath) c.Assert(err, check.NotNil) @@ -416,10 +474,10 @@ func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { // TestDelete checks that the delete operation removes data from the storage // driver func (suite *DriverSuite) TestDelete(c *check.C) { - filename := randomString(32) - contents := []byte(randomString(32)) + filename := randomPath(32) + contents := randomContents(32) - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) @@ -434,7 +492,7 @@ func (suite *DriverSuite) TestDelete(c *check.C) { // TestDeleteNonexistent checks that removing a nonexistent key fails. func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { - filename := randomString(32) + filename := randomPath(32) err := suite.StorageDriver.Delete(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) @@ -442,13 +500,13 @@ func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { // TestDeleteFolder checks that deleting a folder removes all child elements. func (suite *DriverSuite) TestDeleteFolder(c *check.C) { - dirname := randomString(32) - filename1 := randomString(32) - filename2 := randomString(32) - contents := []byte(randomString(32)) + dirname := randomPath(32) + filename1 := randomPath(32) + filename2 := randomPath(32) + filename3 := randomPath(32) + contents := randomContents(32) - defer suite.StorageDriver.Delete(path.Join(dirname, filename1)) - defer suite.StorageDriver.Delete(path.Join(dirname, filename2)) + defer suite.StorageDriver.Delete(firstPart(dirname)) err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents) c.Assert(err, check.IsNil) @@ -456,6 +514,22 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) { err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents) c.Assert(err, check.IsNil) + err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents) + c.Assert(err, check.IsNil) + + err = suite.StorageDriver.Delete(path.Join(dirname, filename1)) + c.Assert(err, check.IsNil) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) + c.Assert(err, check.NotNil) + c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) + c.Assert(err, check.IsNil) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) + c.Assert(err, check.IsNil) + err = suite.StorageDriver.Delete(dirname) c.Assert(err, check.IsNil) @@ -466,22 +540,28 @@ func (suite *DriverSuite) TestDeleteFolder(c *check.C) { _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) + + _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) + c.Assert(err, check.NotNil) + c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestStatCall runs verifies the implementation of the storagedriver's Stat call. func (suite *DriverSuite) TestStatCall(c *check.C) { - content := randomString(4096) - dirPath := randomString(32) - fileName := randomString(32) + content := randomContents(4096) + dirPath := randomPath(32) + fileName := randomFilename(32) filePath := path.Join(dirPath, fileName) + defer suite.StorageDriver.Delete(dirPath) + // Call on non-existent file/dir, check error. fi, err := suite.StorageDriver.Stat(filePath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) c.Assert(fi, check.IsNil) - err = suite.StorageDriver.PutContent(filePath, []byte(content)) + err = suite.StorageDriver.PutContent(filePath, content) c.Assert(err, check.IsNil) // Call on regular file, check results @@ -555,7 +635,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { tfName := path.Base(tf.Name()) defer suite.StorageDriver.Delete(tfName) - contents := []byte(randomString(size)) + contents := randomContents(size) _, err = tf.Write(contents) c.Assert(err, check.IsNil) @@ -578,7 +658,7 @@ func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { } func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) { - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) @@ -590,7 +670,7 @@ func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents } func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) { - defer suite.StorageDriver.Delete(filename) + defer suite.StorageDriver.Delete(firstPart(filename)) nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents)) c.Assert(err, check.IsNil) @@ -606,12 +686,55 @@ func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, c c.Assert(readContents, check.DeepEquals, contents) } -var pathChars = []byte("abcdefghijklmnopqrstuvwxyz") +var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") -func randomString(length int64) string { +func randomPath(length int64) string { + path := "" + for int64(len(path)) < length { + chunkLength := rand.Int63n(length-int64(len(path))) + 1 + chunk := randomFilename(chunkLength) + path += chunk + if length-int64(len(path)) == 1 { + path += randomFilename(1) + } else if length-int64(len(path)) > 1 { + path += "/" + } + } + return path +} + +func randomFilename(length int64) string { b := make([]byte, length) for i := range b { - b[i] = pathChars[rand.Intn(len(pathChars))] + b[i] = filenameChars[rand.Intn(len(filenameChars))] } return string(b) } + +func randomContents(length int64) []byte { + b := make([]byte, length) + for i := range b { + b[i] = byte(rand.Intn(2 << 8)) + } + return b +} + +func firstPart(filePath string) string { + for { + if filePath[len(filePath)-1] == '/' { + filePath = filePath[:len(filePath)-1] + } + + dir, file := path.Split(filePath) + if dir == "" && file == "" { + return "/" + } + if dir == "" { + return file + } + if file == "" { + return dir + } + filePath = dir + } +}