Add Openstack Swift storage driver
Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
This commit is contained in:
parent
5ee441cdc7
commit
ea7c53df08
8 changed files with 828 additions and 0 deletions
3
Godeps/Godeps.json
generated
3
Godeps/Godeps.json
generated
|
@ -81,6 +81,9 @@
|
||||||
{
|
{
|
||||||
"ImportPath": "github.com/stevvooe/resumable",
|
"ImportPath": "github.com/stevvooe/resumable",
|
||||||
"Rev": "51ad44105773cafcbe91927f70ac68e1bf78f8b4"
|
"Rev": "51ad44105773cafcbe91927f70ac68e1bf78f8b4"
|
||||||
|
},
|
||||||
|
"ImportPath": "github.com/lebauce/swift",
|
||||||
|
"Rev": "677cb70f5d40fa1a81ddb32f872615a57bb42381"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ImportPath": "github.com/yvasiyarov/go-metrics",
|
"ImportPath": "github.com/yvasiyarov/go-metrics",
|
||||||
|
|
32
cmd/registry-storagedriver-swift/main.go
Normal file
32
cmd/registry-storagedriver-swift/main.go
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
// +build ignore
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/Sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/docker/distribution/registry/storage/driver/ipc"
|
||||||
|
"github.com/docker/distribution/registry/storage/driver/swift"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An out-of-process Swift driver, intended to be run by ipc.NewDriverClient
|
||||||
|
func main() {
|
||||||
|
parametersBytes := []byte(os.Args[1])
|
||||||
|
var parameters map[string]string
|
||||||
|
err := json.Unmarshal(parametersBytes, ¶meters)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
driver, err := swift.FromParameters(parameters)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ipc.StorageDriverServer(driver); err != nil {
|
||||||
|
logrus.Fatalln(err)
|
||||||
|
}
|
||||||
|
}
|
|
@ -28,6 +28,7 @@ import (
|
||||||
_ "github.com/docker/distribution/registry/storage/driver/inmemory"
|
_ "github.com/docker/distribution/registry/storage/driver/inmemory"
|
||||||
_ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront"
|
_ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront"
|
||||||
_ "github.com/docker/distribution/registry/storage/driver/s3"
|
_ "github.com/docker/distribution/registry/storage/driver/s3"
|
||||||
|
_ "github.com/docker/distribution/registry/storage/driver/swift"
|
||||||
"github.com/docker/distribution/version"
|
"github.com/docker/distribution/version"
|
||||||
gorhandlers "github.com/gorilla/handlers"
|
gorhandlers "github.com/gorilla/handlers"
|
||||||
"github.com/yvasiyarov/gorelic"
|
"github.com/yvasiyarov/gorelic"
|
||||||
|
|
|
@ -49,6 +49,7 @@ This section lists all the registry configuration options. Some options in
|
||||||
the list are mutually exclusive. So, make sure to read the detailed reference
|
the list are mutually exclusive. So, make sure to read the detailed reference
|
||||||
information about each option that appears later in this page.
|
information about each option that appears later in this page.
|
||||||
|
|
||||||
|
<<<<<<< HEAD
|
||||||
version: 0.1
|
version: 0.1
|
||||||
log:
|
log:
|
||||||
level: debug
|
level: debug
|
||||||
|
@ -92,6 +93,14 @@ information about each option that appears later in this page.
|
||||||
poolname: radospool
|
poolname: radospool
|
||||||
username: radosuser
|
username: radosuser
|
||||||
chunksize: 4194304
|
chunksize: 4194304
|
||||||
|
swift:
|
||||||
|
username: username
|
||||||
|
password: password
|
||||||
|
authurl: https://storage.myprovider.com/v2.0
|
||||||
|
tenant: tenantname
|
||||||
|
region: fr
|
||||||
|
container: containername
|
||||||
|
rootdirectory: /swift/object/name/prefix
|
||||||
cache:
|
cache:
|
||||||
blobdescriptor: redis
|
blobdescriptor: redis
|
||||||
maintenance:
|
maintenance:
|
||||||
|
@ -580,6 +589,107 @@ must be set.
|
||||||
|
|
||||||
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
|
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
|
||||||
|
|
||||||
|
### Openstack Swift
|
||||||
|
|
||||||
|
This storage backend uses Openstack Swift object storage.
|
||||||
|
|
||||||
|
<table>
|
||||||
|
<tr>
|
||||||
|
<th>Parameter</th>
|
||||||
|
<th>Required</th>
|
||||||
|
<th>Description</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>authurl</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
yes
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
URL for obtaining an auth token.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>username</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
yes
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
Your Openstack user name.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>password</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
yes
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
Your Openstack password.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>region</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
no
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
The Openstack region in which your container exists.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>container</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
yes
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
The container name in which you want to store the registry's data.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>tenant</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
no
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
Your Openstack tenant name.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>chunksize</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
no
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
Size of the data segments for the Swift Dynamic Large Objects. This value should be a number (defaults to 5M).
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<code>rootdirectory</code>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
no
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
This is a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary.
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
|
||||||
## auth
|
## auth
|
||||||
|
|
||||||
auth:
|
auth:
|
||||||
|
|
21
docs/storage-drivers/swift.md
Normal file
21
docs/storage-drivers/swift.md
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
# Openstack Swift storage driver
|
||||||
|
|
||||||
|
An implementation of the `storagedriver.StorageDriver` interface which uses Openstack Swift for object storage.
|
||||||
|
|
||||||
|
## Parameters
|
||||||
|
|
||||||
|
`authurl`: URL for obtaining an auth token.
|
||||||
|
|
||||||
|
`username`: Your Openstack user name.
|
||||||
|
|
||||||
|
`password`: Your Openstack password.
|
||||||
|
|
||||||
|
`container`: The name of your Swift container where you wish to store objects. An additional container - named `<container>_segments` to store the data will be used. The driver will try to create both containers during its initialization.
|
||||||
|
|
||||||
|
`tenant`: (optional) Your Openstack tenant name.
|
||||||
|
|
||||||
|
`region`: (optional) The name of the Openstack region in which you would like to store objects (for example `fr`).
|
||||||
|
|
||||||
|
`chunksize`: (optional) The segment size for Dynamic Large Objects uploads (performed by WriteStream) to swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift.
|
||||||
|
|
||||||
|
`rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (container root).
|
|
@ -23,6 +23,7 @@ This storage driver package comes bundled with several drivers:
|
||||||
- [s3](storage-drivers/s3.md): A driver storing objects in an Amazon Simple Storage Solution (S3) bucket.
|
- [s3](storage-drivers/s3.md): A driver storing objects in an Amazon Simple Storage Solution (S3) bucket.
|
||||||
- [azure](storage-drivers/azure.md): A driver storing objects in [Microsoft Azure Blob Storage](http://azure.microsoft.com/en-us/services/storage/).
|
- [azure](storage-drivers/azure.md): A driver storing objects in [Microsoft Azure Blob Storage](http://azure.microsoft.com/en-us/services/storage/).
|
||||||
- [rados](storage-drivers/rados.md): A driver storing objects in a [Ceph Object Storage](http://ceph.com/docs/master/rados/) pool.
|
- [rados](storage-drivers/rados.md): A driver storing objects in a [Ceph Object Storage](http://ceph.com/docs/master/rados/) pool.
|
||||||
|
- [swift](storage-drivers/swift): A driver storing objects in Openstack Swift.
|
||||||
|
|
||||||
## Storage Driver API
|
## Storage Driver API
|
||||||
|
|
||||||
|
|
519
registry/storage/driver/swift/swift.go
Normal file
519
registry/storage/driver/swift/swift.go
Normal file
|
@ -0,0 +1,519 @@
|
||||||
|
// Package swift provides a storagedriver.StorageDriver implementation to
|
||||||
|
// store blobs in Openstack Swift object storage.
|
||||||
|
//
|
||||||
|
// This package leverages the ncw/swift client library for interfacing with
|
||||||
|
// Swift.
|
||||||
|
//
|
||||||
|
// Because Swift is a key, value store the Stat call does not support last modification
|
||||||
|
// time for directories (directories are an abstraction for key, value stores)
|
||||||
|
package swift
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
gopath "path"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lebauce/swift"
|
||||||
|
|
||||||
|
"github.com/docker/distribution/context"
|
||||||
|
storagedriver "github.com/docker/distribution/registry/storage/driver"
|
||||||
|
"github.com/docker/distribution/registry/storage/driver/base"
|
||||||
|
"github.com/docker/distribution/registry/storage/driver/factory"
|
||||||
|
)
|
||||||
|
|
||||||
|
const driverName = "swift"
|
||||||
|
|
||||||
|
const defaultChunkSize = 5 * 1024 * 1024
|
||||||
|
|
||||||
|
//DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
|
||||||
|
type DriverParameters struct {
|
||||||
|
Username string
|
||||||
|
Password string
|
||||||
|
AuthURL string
|
||||||
|
Tenant string
|
||||||
|
Region string
|
||||||
|
Container string
|
||||||
|
Prefix string
|
||||||
|
ChunkSize int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type swiftInfo map[string]interface{}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
factory.Register(driverName, &swiftDriverFactory{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// swiftDriverFactory implements the factory.StorageDriverFactory interface
|
||||||
|
type swiftDriverFactory struct{}
|
||||||
|
|
||||||
|
func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
|
||||||
|
return FromParameters(parameters)
|
||||||
|
}
|
||||||
|
|
||||||
|
type driver struct {
|
||||||
|
Conn swift.Connection
|
||||||
|
Container string
|
||||||
|
Prefix string
|
||||||
|
BulkDeleteSupport bool
|
||||||
|
ChunkSize int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type baseEmbed struct {
|
||||||
|
base.Base
|
||||||
|
}
|
||||||
|
|
||||||
|
// Driver is a storagedriver.StorageDriver implementation backed by Amazon Swift
|
||||||
|
// Objects are stored at absolute keys in the provided bucket.
|
||||||
|
type Driver struct {
|
||||||
|
baseEmbed
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromParameters constructs a new Driver with a given parameters map
|
||||||
|
// Required parameters:
|
||||||
|
// - username
|
||||||
|
// - password
|
||||||
|
// - authurl
|
||||||
|
// - container
|
||||||
|
func FromParameters(parameters map[string]interface{}) (*Driver, error) {
|
||||||
|
username, ok := parameters["username"]
|
||||||
|
if !ok || fmt.Sprint(username) == "" {
|
||||||
|
return nil, fmt.Errorf("No username parameter provided")
|
||||||
|
}
|
||||||
|
password, ok := parameters["password"]
|
||||||
|
if !ok || fmt.Sprint(password) == "" {
|
||||||
|
return nil, fmt.Errorf("No password parameter provided")
|
||||||
|
}
|
||||||
|
authURL, ok := parameters["authurl"]
|
||||||
|
if !ok || fmt.Sprint(authURL) == "" {
|
||||||
|
return nil, fmt.Errorf("No container parameter provided")
|
||||||
|
}
|
||||||
|
container, ok := parameters["container"]
|
||||||
|
if !ok || fmt.Sprint(container) == "" {
|
||||||
|
return nil, fmt.Errorf("No container parameter provided")
|
||||||
|
}
|
||||||
|
tenant, ok := parameters["tenant"]
|
||||||
|
if !ok {
|
||||||
|
tenant = ""
|
||||||
|
}
|
||||||
|
region, ok := parameters["region"]
|
||||||
|
if !ok {
|
||||||
|
region = ""
|
||||||
|
}
|
||||||
|
rootDirectory, ok := parameters["rootdirectory"]
|
||||||
|
if !ok {
|
||||||
|
rootDirectory = ""
|
||||||
|
}
|
||||||
|
chunkSize := int64(defaultChunkSize)
|
||||||
|
chunkSizeParam, ok := parameters["chunksize"]
|
||||||
|
if ok {
|
||||||
|
chunkSize, ok = chunkSizeParam.(int64)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("The chunksize parameter should be a number")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
params := DriverParameters{
|
||||||
|
fmt.Sprint(username),
|
||||||
|
fmt.Sprint(password),
|
||||||
|
fmt.Sprint(authURL),
|
||||||
|
fmt.Sprint(tenant),
|
||||||
|
fmt.Sprint(region),
|
||||||
|
fmt.Sprint(container),
|
||||||
|
fmt.Sprint(rootDirectory),
|
||||||
|
chunkSize,
|
||||||
|
}
|
||||||
|
|
||||||
|
return New(params)
|
||||||
|
}
|
||||||
|
|
||||||
|
// New constructs a new Driver with the given Openstack Swift credentials and container name
|
||||||
|
func New(params DriverParameters) (*Driver, error) {
|
||||||
|
ct := swift.Connection{
|
||||||
|
UserName: params.Username,
|
||||||
|
ApiKey: params.Password,
|
||||||
|
AuthUrl: params.AuthURL,
|
||||||
|
Region: params.Region,
|
||||||
|
UserAgent: "distribution",
|
||||||
|
Tenant: params.Tenant,
|
||||||
|
ConnectTimeout: 60 * time.Second,
|
||||||
|
Timeout: 15 * 60 * time.Second,
|
||||||
|
}
|
||||||
|
err := ct.Authenticate()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("Swift authentication failed: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ct.ContainerCreate(params.Container, nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ct.ContainerCreate(params.Container + "_segments", nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container + "_segments", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d := &driver{
|
||||||
|
Conn: ct,
|
||||||
|
Container: params.Container,
|
||||||
|
Prefix: params.Prefix,
|
||||||
|
BulkDeleteSupport: detectBulkDelete(params.AuthURL),
|
||||||
|
ChunkSize: params.ChunkSize,
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Driver{
|
||||||
|
baseEmbed: baseEmbed{
|
||||||
|
Base: base.Base{
|
||||||
|
StorageDriver: d,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Implement the storagedriver.StorageDriver interface
|
||||||
|
|
||||||
|
func (d *driver) Name() string {
|
||||||
|
return driverName
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetContent retrieves the content stored at "path" as a []byte.
|
||||||
|
func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
|
||||||
|
content, err := d.Conn.ObjectGetBytes(d.Container, d.swiftPath(path))
|
||||||
|
if err != nil {
|
||||||
|
return nil, parseError(path, err)
|
||||||
|
}
|
||||||
|
return content, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutContent stores the []byte content at a location designated by "path".
|
||||||
|
func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
|
||||||
|
if dir, err := d.createParentFolder(path); err != nil {
|
||||||
|
return parseError(dir, err)
|
||||||
|
}
|
||||||
|
err := d.Conn.ObjectPutBytes(d.Container, d.swiftPath(path),
|
||||||
|
contents, d.getContentType())
|
||||||
|
return parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
|
||||||
|
// given byte offset.
|
||||||
|
func (d *driver) ReadStream(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
|
||||||
|
headers := make(swift.Headers)
|
||||||
|
headers["Range"] = "bytes=" + strconv.FormatInt(offset, 10) + "-"
|
||||||
|
|
||||||
|
file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 416 {
|
||||||
|
return ioutil.NopCloser(bytes.NewReader(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteStream stores the contents of the provided io.Reader at a
|
||||||
|
// location designated by the given path. The driver will know it has
|
||||||
|
// received the full contents when the reader returns io.EOF. The number
|
||||||
|
// of successfully READ bytes will be returned, even if an error is
|
||||||
|
// returned. May be used to resume writing a stream by providing a nonzero
|
||||||
|
// offset. Offsets past the current size will write from the position
|
||||||
|
// beyond the end of the file.
|
||||||
|
func (d *driver) WriteStream(ctx context.Context, path string, offset int64, reader io.Reader) (int64, error) {
|
||||||
|
var (
|
||||||
|
segments []swift.Object
|
||||||
|
paddingReader io.Reader
|
||||||
|
)
|
||||||
|
|
||||||
|
partNumber := int64(1)
|
||||||
|
bytesRead := int64(0)
|
||||||
|
currentLength := int64(0)
|
||||||
|
zeroBuf := make([]byte, d.ChunkSize)
|
||||||
|
segmentsContainer := d.Container + "_segments"
|
||||||
|
cursor := int64(0)
|
||||||
|
|
||||||
|
getSegment := func() string {
|
||||||
|
return d.swiftPath(path) + "/" + fmt.Sprintf("%016d", partNumber)
|
||||||
|
}
|
||||||
|
|
||||||
|
max := func(a int64, b int64) int64 {
|
||||||
|
if a > b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
info, _, err := d.Conn.Object(d.Container, d.swiftPath(path))
|
||||||
|
if err != nil {
|
||||||
|
if swiftErr, ok := err.(*swift.Error); ok {
|
||||||
|
if swiftErr.StatusCode == 404 {
|
||||||
|
// Create a object manifest
|
||||||
|
if dir, err := d.createParentFolder(path); err != nil {
|
||||||
|
return bytesRead, parseError(dir, err)
|
||||||
|
}
|
||||||
|
headers := make(swift.Headers)
|
||||||
|
headers["X-Object-Manifest"] = segmentsContainer + "/" + d.swiftPath(path)
|
||||||
|
manifest, err := d.Conn.ObjectCreate(d.Container, d.swiftPath(path), false, "",
|
||||||
|
d.getContentType(), headers)
|
||||||
|
manifest.Close()
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// The manifest already exists. Get all the segments
|
||||||
|
currentLength = info.Bytes
|
||||||
|
headers := make(swift.Headers)
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
opts := &swift.ObjectsOpts{Prefix: d.swiftPath(path), Headers: headers}
|
||||||
|
segments, err = d.Conn.Objects(d.Container + "_segments", opts)
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// First, we skip the existing segments that are not modified by this call
|
||||||
|
for i := range segments {
|
||||||
|
if offset < cursor + segments[i].Bytes {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
cursor += segments[i].Bytes
|
||||||
|
partNumber++
|
||||||
|
}
|
||||||
|
|
||||||
|
// We reached the end of the file but we haven't reached 'offset' yet
|
||||||
|
// Therefore we add blocks of zeros
|
||||||
|
if offset >= currentLength {
|
||||||
|
for offset - currentLength >= d.ChunkSize {
|
||||||
|
// Insert a block a zero
|
||||||
|
d.Conn.ObjectPut(segmentsContainer, getSegment(),
|
||||||
|
bytes.NewReader(zeroBuf), false, "",
|
||||||
|
d.getContentType(), nil)
|
||||||
|
currentLength += d.ChunkSize
|
||||||
|
partNumber++
|
||||||
|
}
|
||||||
|
|
||||||
|
cursor = currentLength
|
||||||
|
paddingReader = bytes.NewReader(zeroBuf)
|
||||||
|
} else {
|
||||||
|
// Offset is inside the current segment : we need to read the
|
||||||
|
// data from the beginning of the segment to offset
|
||||||
|
paddingReader, _, err = d.Conn.ObjectOpen(segmentsContainer, getSegment(), false, nil)
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(getSegment(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
multi := io.MultiReader(
|
||||||
|
io.LimitReader(paddingReader, offset - cursor),
|
||||||
|
io.LimitReader(reader, d.ChunkSize - (offset - cursor)),
|
||||||
|
)
|
||||||
|
|
||||||
|
for {
|
||||||
|
currentSegment, err := d.Conn.ObjectCreate(segmentsContainer, getSegment(), false, "", d.getContentType(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err := io.Copy(currentSegment, multi)
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n < d.ChunkSize {
|
||||||
|
// We wrote all the data
|
||||||
|
if cursor + n < currentLength {
|
||||||
|
// Copy the end of the chunk
|
||||||
|
headers := make(swift.Headers)
|
||||||
|
headers["Range"] = "bytes=" + strconv.FormatInt(cursor + n, 10) + "-" + strconv.FormatInt(cursor + d.ChunkSize, 10)
|
||||||
|
file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
|
||||||
|
if err != nil {
|
||||||
|
return bytesRead, parseError(path, err)
|
||||||
|
}
|
||||||
|
io.Copy(currentSegment, file)
|
||||||
|
file.Close()
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
currentSegment.Close()
|
||||||
|
bytesRead += n - max(0, offset - cursor)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
currentSegment.Close()
|
||||||
|
bytesRead += n - max(0, offset - cursor)
|
||||||
|
multi = io.MultiReader(io.LimitReader(reader, d.ChunkSize))
|
||||||
|
cursor += d.ChunkSize
|
||||||
|
partNumber++
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytesRead, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat retrieves the FileInfo for the given path, including the current size
|
||||||
|
// in bytes and the creation time.
|
||||||
|
func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
|
||||||
|
info, _, err := d.Conn.Object(d.Container, d.swiftPath(path))
|
||||||
|
if err != nil {
|
||||||
|
return nil, parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fi := storagedriver.FileInfoFields{
|
||||||
|
Path: path,
|
||||||
|
IsDir: info.ContentType == "application/directory",
|
||||||
|
Size: info.Bytes,
|
||||||
|
ModTime: info.LastModified,
|
||||||
|
}
|
||||||
|
|
||||||
|
return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns a list of the objects that are direct descendants of the given path.
|
||||||
|
func (d *driver) List(ctx context.Context, path string) ([]string, error) {
|
||||||
|
prefix := d.swiftPath(path)
|
||||||
|
if prefix != "" {
|
||||||
|
prefix += "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := &swift.ObjectsOpts{
|
||||||
|
Path: prefix,
|
||||||
|
Delimiter: '/',
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := d.Conn.ObjectNames(d.Container, opts)
|
||||||
|
for index, name := range files {
|
||||||
|
files[index] = "/" + strings.TrimSuffix(name, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
return files, parseError(path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Move moves an object stored at sourcePath to destPath, removing the original
|
||||||
|
// object.
|
||||||
|
func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
|
||||||
|
err := d.Conn.ObjectMove(d.Container, d.swiftPath(sourcePath),
|
||||||
|
d.Container, d.swiftPath(destPath))
|
||||||
|
if err != nil {
|
||||||
|
return parseError(sourcePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete recursively deletes all objects stored at "path" and its subpaths.
|
||||||
|
func (d *driver) Delete(ctx context.Context, path string) error {
|
||||||
|
opts := swift.ObjectsOpts{
|
||||||
|
Prefix: d.swiftPath(path),
|
||||||
|
}
|
||||||
|
|
||||||
|
objects, err := d.Conn.ObjectNamesAll(d.Container, &opts)
|
||||||
|
if err != nil {
|
||||||
|
return parseError(path, err)
|
||||||
|
}
|
||||||
|
if len(objects) == 0 {
|
||||||
|
return storagedriver.PathNotFoundError{Path: path}
|
||||||
|
}
|
||||||
|
|
||||||
|
for index, name := range objects {
|
||||||
|
objects[index] = name[len(d.Prefix):]
|
||||||
|
}
|
||||||
|
|
||||||
|
var multiDelete = true
|
||||||
|
if d.BulkDeleteSupport {
|
||||||
|
_, err := d.Conn.BulkDelete(d.Container, objects)
|
||||||
|
multiDelete = err != nil
|
||||||
|
}
|
||||||
|
if multiDelete {
|
||||||
|
for _, name := range objects {
|
||||||
|
if _, headers, err := d.Conn.Object(d.Container, name); err == nil {
|
||||||
|
manifest, ok := headers["X-Object-Manifest"]
|
||||||
|
if ok {
|
||||||
|
components := strings.SplitN(manifest, "/", 2)
|
||||||
|
segContainer := components[0]
|
||||||
|
segments, err := d.Conn.ObjectNamesAll(segContainer, &swift.ObjectsOpts{ Prefix: components[1] })
|
||||||
|
if err != nil {
|
||||||
|
return parseError(name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, s := range segments {
|
||||||
|
if err := d.Conn.ObjectDelete(segContainer, s); err != nil {
|
||||||
|
return parseError(s, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return parseError(name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := d.Conn.ObjectDelete(d.Container, name); err != nil {
|
||||||
|
return parseError(name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
|
||||||
|
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
|
||||||
|
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
|
||||||
|
return "", storagedriver.ErrUnsupportedMethod
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *driver) swiftPath(path string) string {
|
||||||
|
return strings.TrimLeft(strings.TrimRight(d.Prefix, "/")+path, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *driver) createParentFolder(path string) (string, error) {
|
||||||
|
dir := gopath.Dir(path)
|
||||||
|
if dir != "/" {
|
||||||
|
_, _, err := d.Conn.Object(d.Container, d.swiftPath(dir))
|
||||||
|
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 404 {
|
||||||
|
_, err := d.Conn.ObjectPut(d.Container, d.swiftPath(dir), bytes.NewReader(make([]byte, 0)),
|
||||||
|
false, "", "application/directory", nil)
|
||||||
|
if err != nil {
|
||||||
|
return dir, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *driver) getContentType() string {
|
||||||
|
return "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectBulkDelete(authURL string) (bulkDelete bool) {
|
||||||
|
resp, err := http.Get(filepath.Join(authURL, "..", "..") + "/info")
|
||||||
|
if err == nil {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decoder := json.NewDecoder(resp.Body)
|
||||||
|
var infos swiftInfo
|
||||||
|
if decoder.Decode(&infos) == nil {
|
||||||
|
_, bulkDelete = infos["bulk_delete"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseError(path string, err error) error {
|
||||||
|
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 404 {
|
||||||
|
return storagedriver.PathNotFoundError{Path: path}
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
141
registry/storage/driver/swift/swift_test.go
Normal file
141
registry/storage/driver/swift/swift_test.go
Normal file
|
@ -0,0 +1,141 @@
|
||||||
|
package swift
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/lebauce/swift/swifttest"
|
||||||
|
|
||||||
|
"github.com/docker/distribution/context"
|
||||||
|
storagedriver "github.com/docker/distribution/registry/storage/driver"
|
||||||
|
"github.com/docker/distribution/registry/storage/driver/testsuites"
|
||||||
|
|
||||||
|
"gopkg.in/check.v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Hook up gocheck into the "go test" runner.
|
||||||
|
func Test(t *testing.T) { check.TestingT(t) }
|
||||||
|
|
||||||
|
type SwiftDriverConstructor func(rootDirectory string) (*Driver, error)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
var (
|
||||||
|
username string
|
||||||
|
password string
|
||||||
|
authURL string
|
||||||
|
tenant string
|
||||||
|
container string
|
||||||
|
region string
|
||||||
|
prefix string
|
||||||
|
swiftServer *swifttest.SwiftServer
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
if username = os.Getenv("OS_USERNAME"); username == "" {
|
||||||
|
username = os.Getenv("ST_USER")
|
||||||
|
}
|
||||||
|
if password = os.Getenv("OS_PASSWORD"); password == "" {
|
||||||
|
password = os.Getenv("ST_KEY")
|
||||||
|
}
|
||||||
|
if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
|
||||||
|
authURL = os.Getenv("ST_AUTH")
|
||||||
|
}
|
||||||
|
tenant = os.Getenv("OS_TENANT_NAME")
|
||||||
|
container = os.Getenv("OS_CONTAINER_NAME")
|
||||||
|
region = os.Getenv("OS_REGION_NAME")
|
||||||
|
prefix = os.Getenv("OS_CONTAINER_PREFIX")
|
||||||
|
|
||||||
|
if username == "" || password == "" || authURL == "" || container == "" {
|
||||||
|
if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
username = "swifttest"
|
||||||
|
password = "swifttest"
|
||||||
|
authURL = swiftServer.AuthURL
|
||||||
|
container = "test"
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := ioutil.TempDir("", "driver-")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
defer os.Remove(root)
|
||||||
|
|
||||||
|
swiftDriverConstructor := func(rootDirectory string) (*Driver, error) {
|
||||||
|
parameters := DriverParameters{
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
authURL,
|
||||||
|
tenant,
|
||||||
|
region,
|
||||||
|
container,
|
||||||
|
prefix,
|
||||||
|
defaultChunkSize,
|
||||||
|
}
|
||||||
|
|
||||||
|
return New(parameters)
|
||||||
|
}
|
||||||
|
|
||||||
|
skipCheck := func() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
driverConstructor := func() (storagedriver.StorageDriver, error) {
|
||||||
|
return swiftDriverConstructor(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
testsuites.RegisterInProcessSuite(driverConstructor, skipCheck)
|
||||||
|
|
||||||
|
RegisterSwiftDriverSuite(swiftDriverConstructor, skipCheck, swiftServer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func RegisterSwiftDriverSuite(swiftDriverConstructor SwiftDriverConstructor, skipCheck testsuites.SkipCheck,
|
||||||
|
swiftServer *swifttest.SwiftServer) {
|
||||||
|
check.Suite(&SwiftDriverSuite{
|
||||||
|
Constructor: swiftDriverConstructor,
|
||||||
|
SkipCheck: skipCheck,
|
||||||
|
SwiftServer: swiftServer,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type SwiftDriverSuite struct {
|
||||||
|
Constructor SwiftDriverConstructor
|
||||||
|
SwiftServer *swifttest.SwiftServer
|
||||||
|
testsuites.SkipCheck
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *SwiftDriverSuite) SetUpSuite(c *check.C) {
|
||||||
|
if reason := suite.SkipCheck(); reason != "" {
|
||||||
|
c.Skip(reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *SwiftDriverSuite) TestEmptyRootList(c *check.C) {
|
||||||
|
validRoot, err := ioutil.TempDir("", "driver-")
|
||||||
|
c.Assert(err, check.IsNil)
|
||||||
|
defer os.Remove(validRoot)
|
||||||
|
|
||||||
|
rootedDriver, err := suite.Constructor(validRoot)
|
||||||
|
c.Assert(err, check.IsNil)
|
||||||
|
emptyRootDriver, err := suite.Constructor("")
|
||||||
|
c.Assert(err, check.IsNil)
|
||||||
|
slashRootDriver, err := suite.Constructor("/")
|
||||||
|
c.Assert(err, check.IsNil)
|
||||||
|
|
||||||
|
filename := "/test"
|
||||||
|
contents := []byte("contents")
|
||||||
|
ctx := context.Background()
|
||||||
|
err = rootedDriver.PutContent(ctx, filename, contents)
|
||||||
|
c.Assert(err, check.IsNil)
|
||||||
|
defer rootedDriver.Delete(ctx, filename)
|
||||||
|
|
||||||
|
keys, err := emptyRootDriver.List(ctx, "/")
|
||||||
|
for _, path := range keys {
|
||||||
|
c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
keys, err = slashRootDriver.List(ctx, "/")
|
||||||
|
for _, path := range keys {
|
||||||
|
c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true)
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue