Move shared checks to base storage driver

This changeset moves common checks from driver implementations into base
package. The Base type can be embedded in a driver implementation, ensuring
that incoming arguments are checked for validity.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day 2015-02-03 16:54:52 -08:00
parent 624fb9b565
commit f265a29f24
4 changed files with 244 additions and 170 deletions

141
storagedriver/base/base.go Normal file
View file

@ -0,0 +1,141 @@
// Package base provides a base implementation of the storage driver that can
// be used to implement common checks. The goal is to increase the amount of
// code sharing.
//
// The canonical approach to use this class is to embed in the exported driver
// struct such that calls are proxied through this implementation. First,
// declare the internal driver, as follows:
//
// type driver struct { ... internal ...}
//
// The resulting type should implement StorageDriver such that it can be the
// target of a Base struct. The exported type can then be declared as follows:
//
// type Driver struct {
// Base
// }
//
// Because Driver embeds Base, it effectively implements Base. If the driver
// needs to intercept a call, before going to base, Driver should implement
// that method. Effectively, Driver can intercept calls before coming in and
// driver implements the actual logic.
//
// To further shield the embed from other packages, it is recommended to
// employ a private embed struct:
//
// type baseEmbed struct {
// base.Base
// }
//
// Then, declare driver to embed baseEmbed, rather than Base directly:
//
// type Driver struct {
// baseEmbed
// }
//
// The type now implements StorageDriver, proxying through Base, without
// exporting an unnessecary field.
package base
import (
"io"
"github.com/docker/distribution/storagedriver"
)
// Base provides a wrapper around a storagedriver implementation that provides
// common path and bounds checking.
type Base struct {
storagedriver.StorageDriver
}
// GetContent wraps GetContent of underlying storage driver.
func (base *Base) GetContent(path string) ([]byte, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.GetContent(path)
}
// PutContent wraps PutContent of underlying storage driver.
func (base *Base) PutContent(path string, content []byte) error {
if !storagedriver.PathRegexp.MatchString(path) {
return storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.PutContent(path, content)
}
// ReadStream wraps ReadStream of underlying storage driver.
func (base *Base) ReadStream(path string, offset int64) (io.ReadCloser, error) {
if offset < 0 {
return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.ReadStream(path, offset)
}
// WriteStream wraps WriteStream of underlying storage driver.
func (base *Base) WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) {
if offset < 0 {
return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
if !storagedriver.PathRegexp.MatchString(path) {
return 0, storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.WriteStream(path, offset, reader)
}
// Stat wraps Stat of underlying storage driver.
func (base *Base) Stat(path string) (storagedriver.FileInfo, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.Stat(path)
}
// List wraps List of underlying storage driver.
func (base *Base) List(path string) ([]string, error) {
if !storagedriver.PathRegexp.MatchString(path) && path != "/" {
return nil, storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.List(path)
}
// Move wraps Move of underlying storage driver.
func (base *Base) Move(sourcePath string, destPath string) error {
if !storagedriver.PathRegexp.MatchString(sourcePath) {
return storagedriver.InvalidPathError{Path: sourcePath}
} else if !storagedriver.PathRegexp.MatchString(destPath) {
return storagedriver.InvalidPathError{Path: destPath}
}
return base.StorageDriver.Move(sourcePath, destPath)
}
// Delete wraps Delete of underlying storage driver.
func (base *Base) Delete(path string) error {
if !storagedriver.PathRegexp.MatchString(path) {
return storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.Delete(path)
}
// URLFor wraps URLFor of underlying storage driver.
func (base *Base) URLFor(path string, options map[string]interface{}) (string, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return "", storagedriver.InvalidPathError{Path: path}
}
return base.StorageDriver.URLFor(path, options)
}

View file

@ -10,6 +10,7 @@ import (
"time"
"github.com/docker/distribution/storagedriver"
"github.com/docker/distribution/storagedriver/base"
"github.com/docker/distribution/storagedriver/factory"
)
@ -27,12 +28,20 @@ func (factory *filesystemDriverFactory) Create(parameters map[string]interface{}
return FromParameters(parameters), nil
}
// Driver is a storagedriver.StorageDriver implementation backed by a local
// filesystem. All provided paths will be subpaths of the RootDirectory
type Driver struct {
type driver struct {
rootDirectory string
}
type baseEmbed struct {
base.Base
}
// Driver is a storagedriver.StorageDriver implementation backed by a local
// filesystem. All provided paths will be subpaths of the RootDirectory.
type Driver struct {
baseEmbed
}
// FromParameters constructs a new Driver with a given parameters map
// Optional Parameters:
// - rootdirectory
@ -49,17 +58,21 @@ func FromParameters(parameters map[string]interface{}) *Driver {
// New constructs a new Driver with a given rootDirectory
func New(rootDirectory string) *Driver {
return &Driver{rootDirectory}
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: &driver{
rootDirectory: rootDirectory,
},
},
},
}
}
// Implement the storagedriver.StorageDriver interface
// GetContent retrieves the content stored at "path" as a []byte.
func (d *Driver) GetContent(path string) ([]byte, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) GetContent(path string) ([]byte, error) {
rc, err := d.ReadStream(path, 0)
if err != nil {
return nil, err
@ -75,11 +88,7 @@ func (d *Driver) GetContent(path string) ([]byte, error) {
}
// PutContent stores the []byte content at a location designated by "path".
func (d *Driver) PutContent(subPath string, contents []byte) error {
if !storagedriver.PathRegexp.MatchString(subPath) {
return storagedriver.InvalidPathError{Path: subPath}
}
func (d *driver) PutContent(subPath string, contents []byte) error {
if _, err := d.WriteStream(subPath, 0, bytes.NewReader(contents)); err != nil {
return err
}
@ -89,15 +98,7 @@ func (d *Driver) PutContent(subPath string, contents []byte) error {
// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
if offset < 0 {
return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
file, err := os.OpenFile(d.fullPath(path), os.O_RDONLY, 0644)
if err != nil {
if os.IsNotExist(err) {
@ -121,15 +122,7 @@ func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
// WriteStream stores the contents of the provided io.Reader at a location
// designated by the given path.
func (d *Driver) WriteStream(subPath string, offset int64, reader io.Reader) (nn int64, err error) {
if !storagedriver.PathRegexp.MatchString(subPath) {
return 0, storagedriver.InvalidPathError{Path: subPath}
}
if offset < 0 {
return 0, storagedriver.InvalidOffsetError{Path: subPath, Offset: offset}
}
func (d *driver) WriteStream(subPath string, offset int64, reader io.Reader) (nn int64, err error) {
// TODO(stevvooe): This needs to be a requirement.
// if !path.IsAbs(subPath) {
// return fmt.Errorf("absolute path required: %q", subPath)
@ -165,11 +158,7 @@ func (d *Driver) WriteStream(subPath string, offset int64, reader io.Reader) (nn
// Stat retrieves the FileInfo for the given path, including the current size
// in bytes and the creation time.
func (d *Driver) Stat(subPath string) (storagedriver.FileInfo, error) {
if !storagedriver.PathRegexp.MatchString(subPath) {
return nil, storagedriver.InvalidPathError{Path: subPath}
}
func (d *driver) Stat(subPath string) (storagedriver.FileInfo, error) {
fullPath := d.fullPath(subPath)
fi, err := os.Stat(fullPath)
@ -189,11 +178,7 @@ func (d *Driver) Stat(subPath string) (storagedriver.FileInfo, error) {
// List returns a list of the objects that are direct descendants of the given
// path.
func (d *Driver) List(subPath string) ([]string, error) {
if !storagedriver.PathRegexp.MatchString(subPath) && subPath != "/" {
return nil, storagedriver.InvalidPathError{Path: subPath}
}
func (d *driver) List(subPath string) ([]string, error) {
if subPath[len(subPath)-1] != '/' {
subPath += "/"
}
@ -224,13 +209,7 @@ func (d *Driver) List(subPath string) ([]string, error) {
// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (d *Driver) Move(sourcePath string, destPath string) error {
if !storagedriver.PathRegexp.MatchString(sourcePath) {
return storagedriver.InvalidPathError{Path: sourcePath}
} else if !storagedriver.PathRegexp.MatchString(destPath) {
return storagedriver.InvalidPathError{Path: destPath}
}
func (d *driver) Move(sourcePath string, destPath string) error {
source := d.fullPath(sourcePath)
dest := d.fullPath(destPath)
@ -247,11 +226,7 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
}
// Delete recursively deletes all objects stored at "path" and its subpaths.
func (d *Driver) Delete(subPath string) error {
if !storagedriver.PathRegexp.MatchString(subPath) {
return storagedriver.InvalidPathError{Path: subPath}
}
func (d *driver) Delete(subPath string) error {
fullPath := d.fullPath(subPath)
_, err := os.Stat(fullPath)
@ -267,12 +242,12 @@ func (d *Driver) Delete(subPath string) error {
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *Driver) URLFor(path string, options map[string]interface{}) (string, error) {
func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) {
return "", storagedriver.ErrUnsupportedMethod
}
// fullPath returns the absolute path of a key within the Driver's storage.
func (d *Driver) fullPath(subPath string) string {
func (d *driver) fullPath(subPath string) string {
return path.Join(d.rootDirectory, subPath)
}

View file

@ -9,6 +9,7 @@ import (
"time"
"github.com/docker/distribution/storagedriver"
"github.com/docker/distribution/storagedriver/base"
"github.com/docker/distribution/storagedriver/factory"
)
@ -25,31 +26,46 @@ func (factory *inMemoryDriverFactory) Create(parameters map[string]interface{})
return New(), nil
}
// Driver is a storagedriver.StorageDriver implementation backed by a local map.
// Intended solely for example and testing purposes.
type Driver struct {
type driver struct {
root *dir
mutex sync.RWMutex
}
// baseEmbed allows us to hide the Base embed.
type baseEmbed struct {
base.Base
}
// Driver is a storagedriver.StorageDriver implementation backed by a local map.
// Intended solely for example and testing purposes.
type Driver struct {
baseEmbed // embedded, hidden base driver.
}
var _ storagedriver.StorageDriver = &Driver{}
// New constructs a new Driver.
func New() *Driver {
return &Driver{root: &dir{
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: &driver{
root: &dir{
common: common{
p: "/",
mod: time.Now(),
},
}}
},
},
},
},
}
}
// Implement the storagedriver.StorageDriver interface.
// GetContent retrieves the content stored at "path" as a []byte.
func (d *Driver) GetContent(path string) ([]byte, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) GetContent(path string) ([]byte, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
@ -63,11 +79,7 @@ func (d *Driver) GetContent(path string) ([]byte, error) {
}
// PutContent stores the []byte content at a location designated by "path".
func (d *Driver) PutContent(p string, contents []byte) error {
if !storagedriver.PathRegexp.MatchString(p) {
return storagedriver.InvalidPathError{Path: p}
}
func (d *driver) PutContent(p string, contents []byte) error {
d.mutex.Lock()
defer d.mutex.Unlock()
@ -86,11 +98,7 @@ func (d *Driver) PutContent(p string, contents []byte) error {
// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
@ -114,11 +122,7 @@ func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
// WriteStream stores the contents of the provided io.ReadCloser at a location
// designated by the given path.
func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) {
if !storagedriver.PathRegexp.MatchString(path) {
return 0, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) {
d.mutex.Lock()
defer d.mutex.Unlock()
@ -159,11 +163,7 @@ func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (nn in
}
// Stat returns info about the provided path.
func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) Stat(path string) (storagedriver.FileInfo, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
@ -189,11 +189,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
// List returns a list of the objects that are direct descendants of the given
// path.
func (d *Driver) List(path string) ([]string, error) {
if !storagedriver.PathRegexp.MatchString(path) && path != "/" {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) List(path string) ([]string, error) {
d.mutex.RLock()
defer d.mutex.RUnlock()
@ -223,13 +219,7 @@ func (d *Driver) List(path string) ([]string, error) {
// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (d *Driver) Move(sourcePath string, destPath string) error {
if !storagedriver.PathRegexp.MatchString(sourcePath) {
return storagedriver.InvalidPathError{Path: sourcePath}
} else if !storagedriver.PathRegexp.MatchString(destPath) {
return storagedriver.InvalidPathError{Path: destPath}
}
func (d *driver) Move(sourcePath string, destPath string) error {
d.mutex.Lock()
defer d.mutex.Unlock()
@ -245,11 +235,7 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
}
// Delete recursively deletes all objects stored at "path" and its subpaths.
func (d *Driver) Delete(path string) error {
if !storagedriver.PathRegexp.MatchString(path) {
return storagedriver.InvalidPathError{Path: path}
}
func (d *driver) Delete(path string) error {
d.mutex.Lock()
defer d.mutex.Unlock()
@ -266,6 +252,6 @@ func (d *Driver) Delete(path string) error {
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *Driver) URLFor(path string, options map[string]interface{}) (string, error) {
func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) {
return "", storagedriver.ErrUnsupportedMethod
}

View file

@ -27,6 +27,7 @@ import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/docker/distribution/storagedriver"
"github.com/docker/distribution/storagedriver/base"
"github.com/docker/distribution/storagedriver/factory"
)
@ -65,9 +66,7 @@ func (factory *s3DriverFactory) Create(parameters map[string]interface{}) (stora
return FromParameters(parameters)
}
// Driver is a storagedriver.StorageDriver implementation backed by Amazon S3
// Objects are stored at absolute keys in the provided bucket
type Driver struct {
type driver struct {
S3 *s3.S3
Bucket *s3.Bucket
ChunkSize int64
@ -75,6 +74,16 @@ type Driver struct {
RootDirectory string
}
type baseEmbed struct {
base.Base
}
// Driver is a storagedriver.StorageDriver implementation backed by Amazon S3
// Objects are stored at absolute keys in the provided bucket.
type Driver struct {
baseEmbed
}
// FromParameters constructs a new Driver with a given parameters map
// Required parameters:
// - accesskey
@ -209,22 +218,27 @@ func New(params DriverParameters) (*Driver, error) {
// }
// }
return &Driver{
d := &driver{
S3: s3obj,
Bucket: bucket,
ChunkSize: params.ChunkSize,
Encrypt: params.Encrypt,
RootDirectory: params.RootDirectory}, nil
RootDirectory: params.RootDirectory,
}
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: d,
},
},
}, nil
}
// Implement the storagedriver.StorageDriver interface
// GetContent retrieves the content stored at "path" as a []byte.
func (d *Driver) GetContent(path string) ([]byte, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) GetContent(path string) ([]byte, error) {
content, err := d.Bucket.Get(d.s3Path(path))
if err != nil {
return nil, parseError(path, err)
@ -233,25 +247,13 @@ func (d *Driver) GetContent(path string) ([]byte, error) {
}
// PutContent stores the []byte content at a location designated by "path".
func (d *Driver) PutContent(path string, contents []byte) error {
if !storagedriver.PathRegexp.MatchString(path) {
return storagedriver.InvalidPathError{Path: path}
}
func (d *driver) PutContent(path string, contents []byte) error {
return parseError(path, d.Bucket.Put(d.s3Path(path), contents, d.getContentType(), getPermissions(), d.getOptions()))
}
// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
if offset < 0 {
return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
headers := make(http.Header)
headers.Add("Range", "bytes="+strconv.FormatInt(offset, 10)+"-")
@ -273,15 +275,7 @@ func (d *Driver) ReadStream(path string, offset int64) (io.ReadCloser, error) {
// returned. May be used to resume writing a stream by providing a nonzero
// offset. Offsets past the current size will write from the position
// beyond the end of the file.
func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (totalRead int64, err error) {
if !storagedriver.PathRegexp.MatchString(path) {
return 0, storagedriver.InvalidPathError{Path: path}
}
if offset < 0 {
return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
}
func (d *driver) WriteStream(path string, offset int64, reader io.Reader) (totalRead int64, err error) {
partNumber := 1
bytesRead := 0
var putErrChan chan error
@ -556,11 +550,7 @@ func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (total
// Stat retrieves the FileInfo for the given path, including the current size
// in bytes and the creation time.
func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) Stat(path string) (storagedriver.FileInfo, error) {
listResponse, err := d.Bucket.List(d.s3Path(path), "", "", 1)
if err != nil {
return nil, err
@ -593,11 +583,7 @@ func (d *Driver) Stat(path string) (storagedriver.FileInfo, error) {
}
// List returns a list of the objects that are direct descendants of the given path.
func (d *Driver) List(path string) ([]string, error) {
if !storagedriver.PathRegexp.MatchString(path) && path != "/" {
return nil, storagedriver.InvalidPathError{Path: path}
}
func (d *driver) List(path string) ([]string, error) {
if path != "/" && path[len(path)-1] != '/' {
path = path + "/"
}
@ -633,13 +619,7 @@ func (d *Driver) List(path string) ([]string, error) {
// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (d *Driver) Move(sourcePath string, destPath string) error {
if !storagedriver.PathRegexp.MatchString(sourcePath) {
return storagedriver.InvalidPathError{Path: sourcePath}
} else if !storagedriver.PathRegexp.MatchString(destPath) {
return storagedriver.InvalidPathError{Path: destPath}
}
func (d *driver) Move(sourcePath string, destPath string) error {
/* This is terrible, but aws doesn't have an actual move. */
_, err := d.Bucket.PutCopy(d.s3Path(destPath), getPermissions(),
s3.CopyOptions{Options: d.getOptions(), ContentType: d.getContentType()}, d.Bucket.Name+"/"+d.s3Path(sourcePath))
@ -651,11 +631,7 @@ func (d *Driver) Move(sourcePath string, destPath string) error {
}
// Delete recursively deletes all objects stored at "path" and its subpaths.
func (d *Driver) Delete(path string) error {
if !storagedriver.PathRegexp.MatchString(path) {
return storagedriver.InvalidPathError{Path: path}
}
func (d *driver) Delete(path string) error {
listResponse, err := d.Bucket.List(d.s3Path(path), "", "", listMax)
if err != nil || len(listResponse.Contents) == 0 {
return storagedriver.PathNotFoundError{Path: path}
@ -684,11 +660,7 @@ func (d *Driver) Delete(path string) error {
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *Driver) URLFor(path string, options map[string]interface{}) (string, error) {
if !storagedriver.PathRegexp.MatchString(path) {
return "", storagedriver.InvalidPathError{Path: path}
}
func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) {
methodString := "GET"
method, ok := options["method"]
if ok {
@ -710,7 +682,7 @@ func (d *Driver) URLFor(path string, options map[string]interface{}) (string, er
return d.Bucket.SignedURLWithMethod(methodString, d.s3Path(path), expiresTime, nil, nil), nil
}
func (d *Driver) s3Path(path string) string {
func (d *driver) s3Path(path string) string {
return strings.TrimLeft(strings.TrimRight(d.RootDirectory, "/")+path, "/")
}
@ -727,7 +699,7 @@ func hasCode(err error, code string) bool {
return ok && s3err.Code == code
}
func (d *Driver) getOptions() s3.Options {
func (d *driver) getOptions() s3.Options {
return s3.Options{SSE: d.Encrypt}
}
@ -735,6 +707,6 @@ func getPermissions() s3.ACL {
return s3.Private
}
func (d *Driver) getContentType() string {
func (d *driver) getContentType() string {
return "application/octet-stream"
}