Storagedriver: GCS: add chunksize parameter

Signed-off-by: Arthur Baars <arthur@semmle.com>
This commit is contained in:
Arthur Baars 2016-02-14 18:15:15 +00:00 committed by Brian Bland
parent 115a6e5803
commit 9432b18e30
2 changed files with 36 additions and 3 deletions

View file

@ -22,6 +22,7 @@ import (
"math/rand" "math/rand"
"net/http" "net/http"
"net/url" "net/url"
"reflect"
"regexp" "regexp"
"sort" "sort"
"strconv" "strconv"
@ -50,7 +51,7 @@ const (
uploadSessionContentType = "application/x-docker-upload-session" uploadSessionContentType = "application/x-docker-upload-session"
minChunkSize = 256 * 1024 minChunkSize = 256 * 1024
maxChunkSize = 20 * minChunkSize defaultChunkSize = 20 * minChunkSize
maxTries = 5 maxTries = 5
) )
@ -65,6 +66,7 @@ type driverParameters struct {
privateKey []byte privateKey []byte
client *http.Client client *http.Client
rootDirectory string rootDirectory string
chunkSize int
} }
func init() { func init() {
@ -87,6 +89,7 @@ type driver struct {
email string email string
privateKey []byte privateKey []byte
rootDirectory string rootDirectory string
chunkSize int
} }
// FromParameters constructs a new Driver with a given parameters map // FromParameters constructs a new Driver with a given parameters map
@ -103,6 +106,31 @@ func FromParameters(parameters map[string]interface{}) (storagedriver.StorageDri
rootDirectory = "" rootDirectory = ""
} }
chunkSize := defaultChunkSize
chunkSizeParam, ok := parameters["chunksize"]
if ok {
switch v := chunkSizeParam.(type) {
case string:
vv, err := strconv.Atoi(v)
if err != nil {
return nil, fmt.Errorf("chunksize parameter must be an integer, %v invalid", chunkSizeParam)
}
chunkSize = vv
case int, uint, int32, uint32, uint64, int64:
chunkSize = int(reflect.ValueOf(v).Convert(reflect.TypeOf(chunkSize)).Int())
default:
return nil, fmt.Errorf("invalid valud for chunksize: %#v", chunkSizeParam)
}
if chunkSize < minChunkSize {
return nil, fmt.Errorf("The chunksize %#v parameter should be a number that is larger than or equal to %d", chunkSize, minChunkSize)
}
if chunkSize%minChunkSize != 0 {
return nil, fmt.Errorf("chunksize should be a multiple of %d", minChunkSize)
}
}
var ts oauth2.TokenSource var ts oauth2.TokenSource
jwtConf := new(jwt.Config) jwtConf := new(jwt.Config)
if keyfile, ok := parameters["keyfile"]; ok { if keyfile, ok := parameters["keyfile"]; ok {
@ -121,7 +149,6 @@ func FromParameters(parameters map[string]interface{}) (storagedriver.StorageDri
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
params := driverParameters{ params := driverParameters{
@ -130,6 +157,7 @@ func FromParameters(parameters map[string]interface{}) (storagedriver.StorageDri
email: jwtConf.Email, email: jwtConf.Email,
privateKey: jwtConf.PrivateKey, privateKey: jwtConf.PrivateKey,
client: oauth2.NewClient(context.Background(), ts), client: oauth2.NewClient(context.Background(), ts),
chunkSize: chunkSize,
} }
return New(params) return New(params)
@ -141,12 +169,16 @@ func New(params driverParameters) (storagedriver.StorageDriver, error) {
if rootDirectory != "" { if rootDirectory != "" {
rootDirectory += "/" rootDirectory += "/"
} }
if params.chunkSize <= 0 || params.chunkSize%minChunkSize != 0 {
return nil, fmt.Errorf("Invalid chunksize: %d is not a positive multiple of %d", params.chunkSize, minChunkSize)
}
d := &driver{ d := &driver{
bucket: params.bucket, bucket: params.bucket,
rootDirectory: rootDirectory, rootDirectory: rootDirectory,
email: params.email, email: params.email,
privateKey: params.privateKey, privateKey: params.privateKey,
client: params.client, client: params.client,
chunkSize: params.chunkSize,
} }
return &base.Base{ return &base.Base{
@ -263,7 +295,7 @@ func (d *driver) Writer(context ctx.Context, path string, append bool) (storaged
client: d.client, client: d.client,
bucket: d.bucket, bucket: d.bucket,
name: d.pathToKey(path), name: d.pathToKey(path),
buffer: make([]byte, maxChunkSize), buffer: make([]byte, d.chunkSize),
} }
if append { if append {

View file

@ -75,6 +75,7 @@ func init() {
email: email, email: email,
privateKey: privateKey, privateKey: privateKey,
client: oauth2.NewClient(ctx.Background(), ts), client: oauth2.NewClient(ctx.Background(), ts),
chunkSize: defaultChunkSize,
} }
return New(parameters) return New(parameters)