1275 lines
40 KiB
Go
1275 lines
40 KiB
Go
package testsuites
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
|
"gopkg.in/check.v1"
|
|
)
|
|
|
|
// Test hooks up gocheck into the "go test" runner.
|
|
func Test(t *testing.T) { check.TestingT(t) }
|
|
|
|
// RegisterSuite registers an in-process storage driver test suite with
|
|
// the go test runner.
|
|
func RegisterSuite(driverConstructor DriverConstructor, skipCheck SkipCheck) {
|
|
check.Suite(&DriverSuite{
|
|
Constructor: driverConstructor,
|
|
SkipCheck: skipCheck,
|
|
ctx: context.Background(),
|
|
})
|
|
}
|
|
|
|
// SkipCheck is a function used to determine if a test suite should be skipped.
|
|
// If a SkipCheck returns a non-empty skip reason, the suite is skipped with
|
|
// the given reason.
|
|
type SkipCheck func() (reason string)
|
|
|
|
// NeverSkip is a default SkipCheck which never skips the suite.
|
|
var NeverSkip SkipCheck = func() string { return "" }
|
|
|
|
// DriverConstructor is a function which returns a new
|
|
// storagedriver.StorageDriver.
|
|
type DriverConstructor func() (storagedriver.StorageDriver, error)
|
|
|
|
// DriverTeardown is a function which cleans up a suite's
|
|
// storagedriver.StorageDriver.
|
|
type DriverTeardown func() error
|
|
|
|
// DriverSuite is a gocheck test suite designed to test a
|
|
// storagedriver.StorageDriver. The intended way to create a DriverSuite is
|
|
// with RegisterSuite.
|
|
type DriverSuite struct {
|
|
Constructor DriverConstructor
|
|
Teardown DriverTeardown
|
|
SkipCheck
|
|
storagedriver.StorageDriver
|
|
ctx context.Context
|
|
}
|
|
|
|
// SetUpSuite sets up the gocheck test suite.
|
|
func (suite *DriverSuite) SetUpSuite(c *check.C) {
|
|
if reason := suite.SkipCheck(); reason != "" {
|
|
c.Skip(reason)
|
|
}
|
|
d, err := suite.Constructor()
|
|
c.Assert(err, check.IsNil)
|
|
suite.StorageDriver = d
|
|
}
|
|
|
|
// TearDownSuite tears down the gocheck test suite.
|
|
func (suite *DriverSuite) TearDownSuite(c *check.C) {
|
|
if suite.Teardown != nil {
|
|
err := suite.Teardown()
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
// TearDownTest tears down the gocheck test.
|
|
// This causes the suite to abort if any files are left around in the storage
|
|
// driver.
|
|
func (suite *DriverSuite) TearDownTest(c *check.C) {
|
|
files, _ := suite.StorageDriver.List(suite.ctx, "/")
|
|
if len(files) > 0 {
|
|
c.Fatalf("Storage driver did not clean up properly. Offending files: %#v", files)
|
|
}
|
|
}
|
|
|
|
// TestRootExists ensures that all storage drivers have a root path by default.
|
|
func (suite *DriverSuite) TestRootExists(c *check.C) {
|
|
_, err := suite.StorageDriver.List(suite.ctx, "/")
|
|
if err != nil {
|
|
c.Fatalf(`the root path "/" should always exist: %v`, err)
|
|
}
|
|
}
|
|
|
|
// TestValidPaths checks that various valid file paths are accepted by the
|
|
// storage driver.
|
|
func (suite *DriverSuite) TestValidPaths(c *check.C) {
|
|
contents := randomContents(64)
|
|
validFiles := []string{
|
|
"/a",
|
|
"/2",
|
|
"/aa",
|
|
"/a.a",
|
|
"/0-9/abcdefg",
|
|
"/abcdefg/z.75",
|
|
"/abc/1.2.3.4.5-6_zyx/123.z/4",
|
|
"/docker/docker-registry",
|
|
"/123.abc",
|
|
"/abc./abc",
|
|
"/.abc",
|
|
"/a--b",
|
|
"/a-.b",
|
|
"/_.abc",
|
|
"/Docker/docker-registry",
|
|
"/Abc/Cba",
|
|
}
|
|
|
|
for _, filename := range validFiles {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
}
|
|
}
|
|
|
|
func (suite *DriverSuite) deletePath(c *check.C, path string) {
|
|
for tries := 2; tries > 0; tries-- {
|
|
err := suite.StorageDriver.Delete(suite.ctx, path)
|
|
if _, ok := err.(storagedriver.PathNotFoundError); ok {
|
|
err = nil
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
paths, _ := suite.StorageDriver.List(suite.ctx, path)
|
|
if len(paths) == 0 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second * 2)
|
|
}
|
|
}
|
|
|
|
// TestInvalidPaths checks that various invalid file paths are rejected by the
|
|
// storage driver.
|
|
func (suite *DriverSuite) TestInvalidPaths(c *check.C) {
|
|
contents := randomContents(64)
|
|
invalidFiles := []string{
|
|
"",
|
|
"/",
|
|
"abc",
|
|
"123.abc",
|
|
"//bcd",
|
|
"/abc_123/",
|
|
}
|
|
|
|
for _, filename := range invalidFiles {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
// only delete if file was successfully written
|
|
if err == nil {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
}
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
}
|
|
|
|
// TestWriteRead1 tests a simple write-read workflow.
|
|
func (suite *DriverSuite) TestWriteRead1(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("a")
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead2 tests a simple write-read workflow with unicode data.
|
|
func (suite *DriverSuite) TestWriteRead2(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("\xc3\x9f")
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead3 tests a simple write-read workflow with a small string.
|
|
func (suite *DriverSuite) TestWriteRead3(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead4 tests a simple write-read workflow with 1MB of data.
|
|
func (suite *DriverSuite) TestWriteRead4(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage
|
|
// driver safely.
|
|
func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte{0x80, 0x80, 0x80, 0x80}
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestTruncate tests that putting smaller contents than an original file does
|
|
// remove the excess contents.
|
|
func (suite *DriverSuite) TestTruncate(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
|
|
contents = randomContents(1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestReadNonexistent tests reading content from an empty path.
|
|
func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
|
|
filename := randomPath(32)
|
|
_, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestWriteReadStreams1 tests a simple write-read streaming workflow.
|
|
func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("a")
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams2 tests a simple write-read streaming workflow with
|
|
// unicode data.
|
|
func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("\xc3\x9f")
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams3 tests a simple write-read streaming workflow with a
|
|
// small amount of data.
|
|
func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB
|
|
// of data.
|
|
func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the
|
|
// storage driver safely.
|
|
func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte{0x80, 0x80, 0x80, 0x80}
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage
|
|
// driver safely.
|
|
func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) {
|
|
if testing.Short() {
|
|
c.Skip("Skipping test in short mode")
|
|
}
|
|
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
checksum := sha256.New()
|
|
var fileSize int64 = 5 * 1024 * 1024 * 1024
|
|
|
|
contents := newRandReader(fileSize)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
written, err := io.Copy(writer, io.TeeReader(contents, checksum))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(written, check.Equals, fileSize)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
writtenChecksum := sha256.New()
|
|
io.Copy(writtenChecksum, reader)
|
|
|
|
c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil))
|
|
}
|
|
|
|
// TestReaderWithOffset tests that the appropriate data is streamed when
|
|
// reading with a given offset.
|
|
func (suite *DriverSuite) TestReaderWithOffset(c *check.C) {
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
chunkSize := int64(32)
|
|
|
|
contentsChunk1 := randomContents(chunkSize)
|
|
contentsChunk2 := randomContents(chunkSize)
|
|
contentsChunk3 := randomContents(chunkSize)
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
|
|
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err = io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, append(contentsChunk2, contentsChunk3...))
|
|
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*2)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err = io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contentsChunk3)
|
|
|
|
// Ensure we get invalid offset for negative offsets.
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, -1)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{})
|
|
c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1))
|
|
c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename)
|
|
c.Assert(reader, check.IsNil)
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
// Read past the end of the content and make sure we get a reader that
|
|
// returns 0 bytes and io.EOF
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
buf := make([]byte, chunkSize)
|
|
n, err := reader.Read(buf)
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
c.Assert(n, check.Equals, 0)
|
|
|
|
// Check the N-1 boundary condition, ensuring we get 1 byte then io.EOF.
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3-1)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
n, err = reader.Read(buf)
|
|
c.Assert(n, check.Equals, 1)
|
|
|
|
// We don't care whether the io.EOF comes on the this read or the first
|
|
// zero read, but the only error acceptable here is io.EOF.
|
|
if err != nil {
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
}
|
|
|
|
// Any more reads should result in zero bytes and io.EOF
|
|
n, err = reader.Read(buf)
|
|
c.Assert(n, check.Equals, 0)
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
}
|
|
|
|
// TestContinueStreamAppendLarge tests that a stream write can be appended to without
|
|
// corrupting the data with a large chunk size.
|
|
func (suite *DriverSuite) TestContinueStreamAppendLarge(c *check.C) {
|
|
suite.testContinueStreamAppend(c, int64(10*1024*1024))
|
|
}
|
|
|
|
// TestContinueStreamAppendSmall is the same as TestContinueStreamAppendLarge, but only
|
|
// with a tiny chunk size in order to test corner cases for some cloud storage drivers.
|
|
func (suite *DriverSuite) TestContinueStreamAppendSmall(c *check.C) {
|
|
suite.testContinueStreamAppend(c, int64(32))
|
|
}
|
|
|
|
func (suite *DriverSuite) testContinueStreamAppend(c *check.C, chunkSize int64) {
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
contentsChunk1 := randomContents(chunkSize)
|
|
contentsChunk2 := randomContents(chunkSize)
|
|
contentsChunk3 := randomContents(chunkSize)
|
|
|
|
fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, bytes.NewReader(contentsChunk1))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contentsChunk1)))
|
|
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
curSize := writer.Size()
|
|
c.Assert(curSize, check.Equals, int64(len(contentsChunk1)))
|
|
|
|
writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(writer.Size(), check.Equals, curSize)
|
|
|
|
nn, err = io.Copy(writer, bytes.NewReader(contentsChunk2))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contentsChunk2)))
|
|
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
curSize = writer.Size()
|
|
c.Assert(curSize, check.Equals, 2*chunkSize)
|
|
|
|
writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(writer.Size(), check.Equals, curSize)
|
|
|
|
nn, err = io.Copy(writer, bytes.NewReader(fullContents[curSize:]))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(fullContents[curSize:])))
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, fullContents)
|
|
}
|
|
|
|
// TestReadNonexistentStream tests that reading a stream for a nonexistent path
|
|
// fails.
|
|
func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
|
|
filename := randomPath(32)
|
|
|
|
_, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.Reader(suite.ctx, filename, 64)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestList checks the returned list of keys after populating a directory tree.
|
|
func (suite *DriverSuite) TestList(c *check.C) {
|
|
rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
defer suite.deletePath(c, rootDirectory)
|
|
|
|
doesnotexist := path.Join(rootDirectory, "nonexistent")
|
|
_, err := suite.StorageDriver.List(suite.ctx, doesnotexist)
|
|
c.Assert(err, check.Equals, storagedriver.PathNotFoundError{
|
|
Path: doesnotexist,
|
|
DriverName: suite.StorageDriver.Name(),
|
|
})
|
|
|
|
parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
childFiles := make([]string, 50)
|
|
for i := 0; i < len(childFiles); i++ {
|
|
childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
childFiles[i] = childFile
|
|
err := suite.StorageDriver.PutContent(suite.ctx, childFile, randomContents(32))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
sort.Strings(childFiles)
|
|
|
|
keys, err := suite.StorageDriver.List(suite.ctx, "/")
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(keys, check.DeepEquals, []string{rootDirectory})
|
|
|
|
keys, err = suite.StorageDriver.List(suite.ctx, rootDirectory)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(keys, check.DeepEquals, []string{parentDirectory})
|
|
|
|
keys, err = suite.StorageDriver.List(suite.ctx, parentDirectory)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
sort.Strings(keys)
|
|
c.Assert(keys, check.DeepEquals, childFiles)
|
|
|
|
// A few checks to add here (check out #819 for more discussion on this):
|
|
// 1. Ensure that all paths are absolute.
|
|
// 2. Ensure that listings only include direct children.
|
|
// 3. Ensure that we only respond to directory listings that end with a slash (maybe?).
|
|
}
|
|
|
|
// TestMove checks that a moved object no longer exists at the source path and
|
|
// does exist at the destination.
|
|
func (suite *DriverSuite) TestMove(c *check.C) {
|
|
contents := randomContents(32)
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
|
|
defer suite.deletePath(c, firstPart(sourcePath))
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestMoveOverwrite checks that a moved object no longer exists at the source
|
|
// path and overwrites the contents at the destination.
|
|
func (suite *DriverSuite) TestMoveOverwrite(c *check.C) {
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
sourceContents := randomContents(32)
|
|
destContents := randomContents(64)
|
|
|
|
defer suite.deletePath(c, firstPart(sourcePath))
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, sourceContents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, destPath, destContents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, sourceContents)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestMoveNonexistent checks that moving a nonexistent key fails and does not
|
|
// delete the data at the destination path.
|
|
func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
|
|
contents := randomContents(32)
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, destPath, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
}
|
|
|
|
// TestMoveInvalid provides various checks for invalid moves.
|
|
func (suite *DriverSuite) TestMoveInvalid(c *check.C) {
|
|
contents := randomContents(32)
|
|
|
|
// Create a regular file.
|
|
err := suite.StorageDriver.PutContent(suite.ctx, "/notadir", contents)
|
|
c.Assert(err, check.IsNil)
|
|
defer suite.deletePath(c, "/notadir")
|
|
|
|
// Now try to move a non-existent file under it.
|
|
err = suite.StorageDriver.Move(suite.ctx, "/notadir/foo", "/notadir/bar")
|
|
c.Assert(err, check.NotNil) // non-nil error
|
|
}
|
|
|
|
// TestDelete checks that the delete operation removes data from the storage
|
|
// driver
|
|
func (suite *DriverSuite) TestDelete(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestURLFor checks that the URLFor method functions properly, but only if it
|
|
// is implemented
|
|
func (suite *DriverSuite) TestURLFor(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
url, err := suite.StorageDriver.URLFor(suite.ctx, filename, nil)
|
|
if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
|
|
return
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
|
|
response, err := http.Get(url)
|
|
c.Assert(err, check.IsNil)
|
|
defer response.Body.Close()
|
|
|
|
read, err := io.ReadAll(response.Body)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(read, check.DeepEquals, contents)
|
|
|
|
url, err = suite.StorageDriver.URLFor(suite.ctx, filename, map[string]interface{}{"method": http.MethodHead})
|
|
if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
|
|
return
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
|
|
response, _ = http.Head(url)
|
|
c.Assert(response.StatusCode, check.Equals, 200)
|
|
c.Assert(response.ContentLength, check.Equals, int64(32))
|
|
}
|
|
|
|
// TestDeleteNonexistent checks that removing a nonexistent key fails.
|
|
func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
|
|
filename := randomPath(32)
|
|
err := suite.StorageDriver.Delete(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestDeleteFolder checks that deleting a folder removes all child elements.
|
|
func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
|
|
dirname := randomPath(32)
|
|
filename1 := randomPath(32)
|
|
filename2 := randomPath(32)
|
|
filename3 := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(dirname))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename1), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename2), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename3), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, dirname)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestDeleteOnlyDeletesSubpaths checks that deleting path A does not
|
|
// delete path B when A is a prefix of B but B is not a subpath of A (so that
|
|
// deleting "/a" does not delete "/ab"). This matters for services like S3 that
|
|
// do not implement directories.
|
|
func (suite *DriverSuite) TestDeleteOnlyDeletesSubpaths(c *check.C) {
|
|
dirname := randomPath(32)
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(dirname))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename+"suffix"), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname, filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename+"suffix"))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, dirname))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname, filename))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
// TestStatCall runs verifies the implementation of the storagedriver's Stat call.
|
|
func (suite *DriverSuite) TestStatCall(c *check.C) {
|
|
content := randomContents(4096)
|
|
dirPath := randomPath(32)
|
|
fileName := randomFilename(32)
|
|
filePath := path.Join(dirPath, fileName)
|
|
|
|
defer suite.deletePath(c, firstPart(dirPath))
|
|
|
|
// Call on non-existent file/dir, check error.
|
|
fi, err := suite.StorageDriver.Stat(suite.ctx, dirPath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
c.Assert(fi, check.IsNil)
|
|
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
c.Assert(fi, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// Call on regular file, check results
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
c.Assert(fi.Path(), check.Equals, filePath)
|
|
c.Assert(fi.Size(), check.Equals, int64(len(content)))
|
|
c.Assert(fi.IsDir(), check.Equals, false)
|
|
createdTime := fi.ModTime()
|
|
|
|
// Sleep and modify the file
|
|
time.Sleep(time.Second * 10)
|
|
content = randomContents(4096)
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
|
|
c.Assert(err, check.IsNil)
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
time.Sleep(time.Second * 5) // allow changes to propagate (eventual consistency)
|
|
|
|
// Check if the modification time is after the creation time.
|
|
// In case of cloud storage services, storage frontend nodes might have
|
|
// time drift between them, however that should be solved with sleeping
|
|
// before update.
|
|
modTime := fi.ModTime()
|
|
if !modTime.After(createdTime) {
|
|
c.Errorf("modtime (%s) is before the creation time (%s)", modTime, createdTime)
|
|
}
|
|
|
|
// Call on directory (do not check ModTime as dirs don't need to support it)
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, dirPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
c.Assert(fi.Path(), check.Equals, dirPath)
|
|
c.Assert(fi.Size(), check.Equals, int64(0))
|
|
c.Assert(fi.IsDir(), check.Equals, true)
|
|
}
|
|
|
|
// TestPutContentMultipleTimes checks that if storage driver can overwrite the content
|
|
// in the subsequent puts. Validates that PutContent does not have to work
|
|
// with an offset like Writer does and overwrites the file entirely
|
|
// rather than writing the data to the [0,len(data)) of the file.
|
|
func (suite *DriverSuite) TestPutContentMultipleTimes(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(4096)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
contents = randomContents(2048) // upload a different, smaller file
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
// TestConcurrentStreamReads checks that multiple clients can safely read from
|
|
// the same file simultaneously with various offsets.
|
|
func (suite *DriverSuite) TestConcurrentStreamReads(c *check.C) {
|
|
var filesize int64 = 128 * 1024 * 1024
|
|
|
|
if testing.Short() {
|
|
filesize = 10 * 1024 * 1024
|
|
c.Log("Reducing file size to 10MB for short mode")
|
|
}
|
|
|
|
filename := randomPath(32)
|
|
contents := randomContents(filesize)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
readContents := func() {
|
|
defer wg.Done()
|
|
offset := rand.Int63n(int64(len(contents)))
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contents[offset:])
|
|
}
|
|
|
|
wg.Add(10)
|
|
for i := 0; i < 10; i++ {
|
|
go readContents()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// TestConcurrentFileStreams checks that multiple *os.File objects can be passed
|
|
// in to Writer concurrently without hanging.
|
|
func (suite *DriverSuite) TestConcurrentFileStreams(c *check.C) {
|
|
numStreams := 32
|
|
|
|
if testing.Short() {
|
|
numStreams = 8
|
|
c.Log("Reducing number of streams to 8 for short mode")
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
testStream := func(size int64) {
|
|
defer wg.Done()
|
|
suite.testFileStreams(c, size)
|
|
}
|
|
|
|
wg.Add(numStreams)
|
|
for i := numStreams; i > 0; i-- {
|
|
go testStream(int64(numStreams) * 1024 * 1024)
|
|
}
|
|
|
|
wg.Wait()
|
|
}
|
|
|
|
// TODO (brianbland): evaluate the relevancy of this test
|
|
// TestEventualConsistency checks that if stat says that a file is a certain size, then
|
|
// you can freely read from the file (this is the only guarantee that the driver needs to provide)
|
|
// func (suite *DriverSuite) TestEventualConsistency(c *check.C) {
|
|
// if testing.Short() {
|
|
// c.Skip("Skipping test in short mode")
|
|
// }
|
|
//
|
|
// filename := randomPath(32)
|
|
// defer suite.deletePath(c, firstPart(filename))
|
|
//
|
|
// var offset int64
|
|
// var misswrites int
|
|
// var chunkSize int64 = 32
|
|
//
|
|
// for i := 0; i < 1024; i++ {
|
|
// contents := randomContents(chunkSize)
|
|
// read, err := suite.StorageDriver.Writer(suite.ctx, filename, offset, bytes.NewReader(contents))
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// fi, err := suite.StorageDriver.Stat(suite.ctx, filename)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// // We are most concerned with being able to read data as soon as Stat declares
|
|
// // it is uploaded. This is the strongest guarantee that some drivers (that guarantee
|
|
// // at best eventual consistency) absolutely need to provide.
|
|
// if fi.Size() == offset+chunkSize {
|
|
// reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// readContents, err := io.ReadAll(reader)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// c.Assert(readContents, check.DeepEquals, contents)
|
|
//
|
|
// reader.Close()
|
|
// offset += read
|
|
// } else {
|
|
// misswrites++
|
|
// }
|
|
// }
|
|
//
|
|
// if misswrites > 0 {
|
|
// c.Log("There were " + string(misswrites) + " occurrences of a write not being instantly available.")
|
|
// }
|
|
//
|
|
// c.Assert(misswrites, check.Not(check.Equals), 1024)
|
|
// }
|
|
|
|
// BenchmarkPutGetEmptyFiles benchmarks PutContent/GetContent for 0B files
|
|
func (suite *DriverSuite) BenchmarkPutGetEmptyFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 0)
|
|
}
|
|
|
|
// BenchmarkPutGet1KBFiles benchmarks PutContent/GetContent for 1KB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1KBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024)
|
|
}
|
|
|
|
// BenchmarkPutGet1MBFiles benchmarks PutContent/GetContent for 1MB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1MBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024*1024)
|
|
}
|
|
|
|
// BenchmarkPutGet1GBFiles benchmarks PutContent/GetContent for 1GB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1GBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024*1024*1024)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkPutGetFiles(c *check.C, size int64) {
|
|
c.SetBytes(size)
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := 0; i < c.N; i++ {
|
|
filename := path.Join(parentDir, randomPath(32))
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, randomContents(size))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
// BenchmarkStreamEmptyFiles benchmarks Writer/Reader for 0B files
|
|
func (suite *DriverSuite) BenchmarkStreamEmptyFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 0)
|
|
}
|
|
|
|
// BenchmarkStream1KBFiles benchmarks Writer/Reader for 1KB files
|
|
func (suite *DriverSuite) BenchmarkStream1KBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024)
|
|
}
|
|
|
|
// BenchmarkStream1MBFiles benchmarks Writer/Reader for 1MB files
|
|
func (suite *DriverSuite) BenchmarkStream1MBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024*1024)
|
|
}
|
|
|
|
// BenchmarkStream1GBFiles benchmarks Writer/Reader for 1GB files
|
|
func (suite *DriverSuite) BenchmarkStream1GBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024*1024*1024)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkStreamFiles(c *check.C, size int64) {
|
|
c.SetBytes(size)
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := 0; i < c.N; i++ {
|
|
filename := path.Join(parentDir, randomPath(32))
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
written, err := io.Copy(writer, bytes.NewReader(randomContents(size)))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(written, check.Equals, size)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
rc, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
rc.Close()
|
|
}
|
|
}
|
|
|
|
// BenchmarkList5Files benchmarks List for 5 small files
|
|
func (suite *DriverSuite) BenchmarkList5Files(c *check.C) {
|
|
suite.benchmarkListFiles(c, 5)
|
|
}
|
|
|
|
// BenchmarkList50Files benchmarks List for 50 small files
|
|
func (suite *DriverSuite) BenchmarkList50Files(c *check.C) {
|
|
suite.benchmarkListFiles(c, 50)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkListFiles(c *check.C, numFiles int64) {
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := int64(0); i < numFiles; i++ {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
c.ResetTimer()
|
|
for i := 0; i < c.N; i++ {
|
|
files, err := suite.StorageDriver.List(suite.ctx, parentDir)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(int64(len(files)), check.Equals, numFiles)
|
|
}
|
|
}
|
|
|
|
// BenchmarkDelete5Files benchmarks Delete for 5 small files
|
|
func (suite *DriverSuite) BenchmarkDelete5Files(c *check.C) {
|
|
suite.benchmarkDeleteFiles(c, 5)
|
|
}
|
|
|
|
// BenchmarkDelete50Files benchmarks Delete for 50 small files
|
|
func (suite *DriverSuite) BenchmarkDelete50Files(c *check.C) {
|
|
suite.benchmarkDeleteFiles(c, 50)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkDeleteFiles(c *check.C, numFiles int64) {
|
|
for i := 0; i < c.N; i++ {
|
|
parentDir := randomPath(8)
|
|
defer suite.deletePath(c, firstPart(parentDir))
|
|
|
|
c.StopTimer()
|
|
for j := int64(0); j < numFiles; j++ {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
c.StartTimer()
|
|
|
|
// This is the operation we're benchmarking
|
|
err := suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
|
|
tf, err := os.CreateTemp("", "tf")
|
|
c.Assert(err, check.IsNil)
|
|
defer os.Remove(tf.Name())
|
|
defer tf.Close()
|
|
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
contents := randomContents(size)
|
|
|
|
_, err = tf.Write(contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
tf.Sync()
|
|
tf.Seek(0, io.SeekStart)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, tf)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, size)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, bytes.NewReader(contents))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contents)))
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := io.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
var (
|
|
filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
|
|
separatorChars = []byte("._-")
|
|
)
|
|
|
|
func randomPath(length int64) string {
|
|
path := "/"
|
|
for int64(len(path)) < length {
|
|
chunkLength := rand.Int63n(length-int64(len(path))) + 1
|
|
chunk := randomFilename(chunkLength)
|
|
path += chunk
|
|
remaining := length - int64(len(path))
|
|
if remaining == 1 {
|
|
path += randomFilename(1)
|
|
} else if remaining > 1 {
|
|
path += "/"
|
|
}
|
|
}
|
|
return path
|
|
}
|
|
|
|
func randomFilename(length int64) string {
|
|
b := make([]byte, length)
|
|
wasSeparator := true
|
|
for i := range b {
|
|
if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 {
|
|
b[i] = separatorChars[rand.Intn(len(separatorChars))]
|
|
wasSeparator = true
|
|
} else {
|
|
b[i] = filenameChars[rand.Intn(len(filenameChars))]
|
|
wasSeparator = false
|
|
}
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// randomBytes pre-allocates all of the memory sizes needed for the test. If
|
|
// anything panics while accessing randomBytes, just make this number bigger.
|
|
var randomBytes = make([]byte, 128<<20)
|
|
|
|
func init() {
|
|
_, _ = rand.Read(randomBytes) // always returns len(randomBytes) and nil error
|
|
}
|
|
|
|
func randomContents(length int64) []byte {
|
|
return randomBytes[:length]
|
|
}
|
|
|
|
type randReader struct {
|
|
r int64
|
|
m sync.Mutex
|
|
}
|
|
|
|
func (rr *randReader) Read(p []byte) (n int, err error) {
|
|
rr.m.Lock()
|
|
defer rr.m.Unlock()
|
|
|
|
toread := int64(len(p))
|
|
if toread > rr.r {
|
|
toread = rr.r
|
|
}
|
|
n = copy(p, randomContents(toread))
|
|
rr.r -= int64(n)
|
|
|
|
if rr.r <= 0 {
|
|
err = io.EOF
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func newRandReader(n int64) *randReader {
|
|
return &randReader{r: n}
|
|
}
|
|
|
|
func firstPart(filePath string) string {
|
|
if filePath == "" {
|
|
return "/"
|
|
}
|
|
for {
|
|
if filePath[len(filePath)-1] == '/' {
|
|
filePath = filePath[:len(filePath)-1]
|
|
}
|
|
|
|
dir, file := path.Split(filePath)
|
|
if dir == "" && file == "" {
|
|
return "/"
|
|
}
|
|
if dir == "/" || dir == "" {
|
|
return "/" + file
|
|
}
|
|
if file == "" {
|
|
return dir
|
|
}
|
|
filePath = dir
|
|
}
|
|
}
|