alexvanin/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #1401 from BrianBland/s3StorageClass

Browse source 
Adds "storageclass" configuration parameter for S3 driver.
This commit is contained in:
Richard Scothern 2016-02-04 16:44:13 -08:00
commit 2cc6ccbded
3 changed files with 112 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
docs/storage-drivers/s3.md
View file

@ -30,7 +30,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
Your AWS Access Key.
</td>
</tr>
<tr>
<tr>
<td>
<code>secretkey</code>
</td>
@ -41,7 +41,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
Your AWS Secret Key.
</td>
</tr>
<tr>
<tr>
<td>
<code>region</code>
</td>
@ -64,7 +64,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
The bucket name in which you want to store the registry's data.
</td>
</tr>
<tr>
<tr>
<td>
<code>encrypt</code>
</td>
@ -76,7 +76,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
not. A boolean value. The default is false.
</td>
</tr>
<tr>
<tr>
<td>
<code>secure</code>
</td>
@ -88,7 +88,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
default is <code>true</code>.
</td>
</tr>
<tr>
<tr>
<td>
<code>v4auth</code>
</td>
@ -101,7 +101,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
<code>false</code>.
</td>
</tr>
<tr>
<tr>
<td>
<code>chunksize</code>
</td>
@ -113,7 +113,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
should be a number that is larger than 5*1024*1024.
</td>
</tr>
<tr>
<tr>
<td>
<code>rootdirectory</code>
</td>
@ -124,6 +124,17 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
This is a prefix that will be applied to all S3 keys to allow you to segment data in your bucket if necessary.
</td>
</tr>
<tr>
<td>
<code>storageclass</code>
</td>
<td>
no
</td>
<td>
The S3 storage class applied to each registry file. The default value is STANDARD.
</td>
</tr>
</table>
@ -147,6 +158,8 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
`rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root).
`storageclass`: (optional) The storage class applied to each registry file. Defaults to STANDARD. Valid options are STANDARD and REDUCED_REDUNDANCY.
# CloudFront as Middleware with S3 backend
## Use Case

26
registry/storage/driver/s3/s3.go
View file

@ -1,7 +1,7 @@
// Package s3 provides a storagedriver.StorageDriver implementation to
// store blobs in Amazon S3 cloud storage.
//
// This package leverages the AdRoll/goamz client library for interfacing with
// This package leverages the docker/goamz client library for interfacing with
// s3.
//
// Because s3 is a key, value store the Stat call does not support last modification
@ -59,6 +59,7 @@ type DriverParameters struct {
V4Auth bool
ChunkSize int64
RootDirectory string
StorageClass s3.StorageClass
UserAgent string
}
@ -79,6 +80,7 @@ type driver struct {
ChunkSize int64
Encrypt bool
RootDirectory string
StorageClass s3.StorageClass
pool sync.Pool // pool []byte buffers used for WriteStream
zeros []byte // shared, zero-valued buffer used for WriteStream
@ -183,6 +185,21 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
rootDirectory = ""
}
storageClass := s3.StandardStorage
storageClassParam, ok := parameters["storageclass"]
if ok {
storageClassString, ok := storageClassParam.(string)
if !ok {
return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam)
}
// All valid storage class parameters are UPPERCASE, so be a bit more flexible here
storageClassCasted := s3.StorageClass(strings.ToUpper(storageClassString))
if storageClassCasted != s3.StandardStorage && storageClassCasted != s3.ReducedRedundancy {
return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam)
}
storageClass = storageClassCasted
}
userAgent, ok := parameters["useragent"]
if !ok {
userAgent = ""
@ -198,6 +215,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
v4AuthBool,
chunkSize,
fmt.Sprint(rootDirectory),
storageClass,
fmt.Sprint(userAgent),
}
@ -259,6 +277,7 @@ func New(params DriverParameters) (*Driver, error) {
ChunkSize: params.ChunkSize,
Encrypt: params.Encrypt,
RootDirectory: params.RootDirectory,
StorageClass: params.StorageClass,
zeros: make([]byte, params.ChunkSize),
}
@ -826,7 +845,10 @@ func hasCode(err error, code string) bool {
}
func (d *driver) getOptions() s3.Options {
return s3.Options{SSE: d.Encrypt}
return s3.Options{
SSE: d.Encrypt,
StorageClass: d.StorageClass,
}
}
func getPermissions() s3.ACL {

74
registry/storage/driver/s3/s3_test.go
View file

@ -10,6 +10,7 @@ import (
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/testsuites"
"github.com/docker/goamz/aws"
"github.com/docker/goamz/s3"
"gopkg.in/check.v1"
)
@ -17,7 +18,7 @@ import (
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
var s3DriverConstructor func(rootDirectory string) (*Driver, error)
var s3DriverConstructor func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error)
var skipS3 func() string
func init() {
@ -34,7 +35,7 @@ func init() {
}
defer os.Remove(root)
s3DriverConstructor = func(rootDirectory string) (*Driver, error) {
s3DriverConstructor = func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error) {
encryptBool := false
if encrypt != "" {
encryptBool, err = strconv.ParseBool(encrypt)
@ -69,6 +70,7 @@ func init() {
v4AuthBool,
minChunkSize,
rootDirectory,
storageClass,
"",
}
@ -84,7 +86,7 @@ func init() {
}
testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
return s3DriverConstructor(root)
return s3DriverConstructor(root, s3.StandardStorage)
}, skipS3)
}
@ -99,17 +101,17 @@ func TestEmptyRootList(t *testing.T) {
}
defer os.Remove(validRoot)
rootedDriver, err := s3DriverConstructor(validRoot)
rootedDriver, err := s3DriverConstructor(validRoot, s3.StandardStorage)
if err != nil {
t.Fatalf("unexpected error creating rooted driver: %v", err)
}
emptyRootDriver, err := s3DriverConstructor("")
emptyRootDriver, err := s3DriverConstructor("", s3.StandardStorage)
if err != nil {
t.Fatalf("unexpected error creating empty root driver: %v", err)
}
slashRootDriver, err := s3DriverConstructor("/")
slashRootDriver, err := s3DriverConstructor("/", s3.StandardStorage)
if err != nil {
t.Fatalf("unexpected error creating slash root driver: %v", err)
}
@ -137,3 +139,63 @@ func TestEmptyRootList(t *testing.T) {
}
}
}
func TestStorageClass(t *testing.T) {
if skipS3() != "" {
t.Skip(skipS3())
}
rootDir, err := ioutil.TempDir("", "driver-")
if err != nil {
t.Fatalf("unexpected error creating temporary directory: %v", err)
}
defer os.Remove(rootDir)
standardDriver, err := s3DriverConstructor(rootDir, s3.StandardStorage)
if err != nil {
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
}
rrDriver, err := s3DriverConstructor(rootDir, s3.ReducedRedundancy)
if err != nil {
t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
}
standardFilename := "/test-standard"
rrFilename := "/test-rr"
contents := []byte("contents")
ctx := context.Background()
err = standardDriver.PutContent(ctx, standardFilename, contents)
if err != nil {
t.Fatalf("unexpected error creating content: %v", err)
}
defer standardDriver.Delete(ctx, standardFilename)
err = rrDriver.PutContent(ctx, rrFilename, contents)
if err != nil {
t.Fatalf("unexpected error creating content: %v", err)
}
defer rrDriver.Delete(ctx, rrFilename)
standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
resp, err := standardDriverUnwrapped.Bucket.GetResponse(standardDriverUnwrapped.s3Path(standardFilename))
if err != nil {
t.Fatalf("unexpected error retrieving standard storage file: %v", err)
}
defer resp.Body.Close()
// Amazon only populates this header value for non-standard storage classes
if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != "" {
t.Fatalf("unexpected storage class for standard file: %v", storageClass)
}
rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
resp, err = rrDriverUnwrapped.Bucket.GetResponse(rrDriverUnwrapped.s3Path(rrFilename))
if err != nil {
t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
}
defer resp.Body.Close()
if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != string(s3.ReducedRedundancy) {
t.Fatalf("unexpected storage class for standard file: %v", storageClass)
}
}