Merge pull request #652 from BrianBland/configuration

NG: Configuration parser

configuration/README.md

@@ -0,0 +1,73 @@
+Docker-Registry Configuration
+=============================
+
+This document describes the registry configuration model and how to specify a custom configuration with a configuration file and/or environment variables.
+
+Semantic-ish Versioning
+-----------------------
+
+The configuration file is designed with versioning in mind, such that most upgrades will not require a change in configuration files, and such that configuration files can be "upgraded" from one version to another.
+
+The version is specified as a string of the form `MajorVersion.MinorVersion`, where MajorVersion and MinorVersion are both non-negative integer values. Much like [semantic versioning](http://semver.org/), minor version increases denote inherently backwards-compatible changes, such as the addition of optional fields, whereas major version increases denote a restructuring, such as renaming fields or adding required fields. Because of the explicit version definition in the configuration file, it should be possible to parse old configuration files and port them to the current configuration version, although this is not guaranteed for all future versions.
+
+File Structure (as of Version 0.1)
+------------------------------------
+
+The configuration structure is defined by the `Configuration` struct in `configuration.go`, and is best described by the following two examples:
+
+```yaml
+version: 0.1
+loglevel: info
+storage:
+  s3:
+    region: us-east-1
+    bucket: my-bucket
+    rootpath: /registry
+    encrypt: true
+    secure: false
+    accesskey: SAMPLEACCESSKEY
+    secretkey: SUPERSECRET
+    host: ~
+    port: ~
+```
+
+```yaml
+version: 0.1
+loglevel: debug
+storage: inmemory
+```
+
+### version
+The version is expected to remain a top-level field, as to allow for a consistent version check before parsing the remainder of the configuration file.
+
+### loglevel
+This specifies the log level of the registry.
+
+Supported values:
+* `error`
+* `warn`
+* `info`
+* `debug`
+
+### storage
+This specifies the storage driver, and may be provided either as a string (only the driver type) or as a driver name with a parameters map, as seen in the first example above.
+
+The parameters map will be passed into the factory constructor of the given storage driver type.
+
+### Notes
+
+All keys in the configuration file **must** be provided as a string of lowercase letters and numbers only, and values must be string-like (booleans and numerical values are fine to parse as strings).
+
+Environment Variables
+---------------------
+
+To support the workflow of running a docker registry from a standard container without having to modify configuration files, the registry configuration also supports environment variables for overriding fields.
+
+Any configuration field other than version can be replaced by providing an environment variable of the following form: `REGISTRY_<uppercase key>[_<uppercase key>]...`.
+
+For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
+
+### Notes
+If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
+
+By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.

configuration/configuration.go

@@ -0,0 +1,266 @@
+package configuration
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"gopkg.in/BrianBland/yaml.v2"
+)
+
+// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
+// optionally modified by environment variables
+type Configuration struct {
+	// Version is the version which defines the format of the rest of the configuration
+	Version  Version  `yaml:"version"`
+
+	// Loglevel is the level at which registry operations are logged
+	Loglevel Loglevel `yaml:"loglevel"`
+
+	// Storage is the configuration for the registry's storage driver
+	Storage  Storage  `yaml:"storage"`
+}
+
+// v_0_1_Configuration is a Version 0.1 Configuration struct
+// This is currently aliased to Configuration, as it is the current version
+type v_0_1_Configuration Configuration
+
+// Version is a major/minor version pair of the form Major.Minor
+// Major version upgrades indicate structure or type changes
+// Minor version upgrades should be strictly additive
+type Version string
+
+// MajorMinorVersion constructs a Version from its Major and Minor components
+func MajorMinorVersion(major, minor uint) Version {
+	return Version(fmt.Sprintf("%d.%d", major, minor))
+}
+
+func (version Version) major() (uint, error) {
+	majorPart := strings.Split(string(version), ".")[0]
+	major, err := strconv.ParseUint(majorPart, 10, 0)
+	return uint(major), err
+}
+
+// Major returns the major version portion of a Version
+func (version Version) Major() uint {
+	major, _ := version.major()
+	return major
+}
+
+func (version Version) minor() (uint, error) {
+	minorPart := strings.Split(string(version), ".")[1]
+	minor, err := strconv.ParseUint(minorPart, 10, 0)
+	return uint(minor), err
+}
+
+// Minor returns the minor version portion of a Version
+func (version Version) Minor() uint {
+	minor, _ := version.minor()
+	return minor
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface
+// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
+func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var versionString string
+	err := unmarshal(&versionString)
+	if err != nil {
+		return err
+	}
+
+	newVersion := Version(versionString)
+	if _, err := newVersion.major(); err != nil {
+		return err
+	}
+
+	if _, err := newVersion.minor(); err != nil {
+		return err
+	}
+
+	*version = newVersion
+	return nil
+}
+
+// CurrentVersion is the most recent Version that can be parsed
+var CurrentVersion = MajorMinorVersion(0, 1)
+
+// Loglevel is the level at which operations are logged
+// This can be error, warn, info, or debug
+type Loglevel string
+
+// UnmarshalYAML implements the yaml.Umarshaler interface
+// Unmarshals a string into a Loglevel, lowercasing the string and validating that it represents a
+// valid loglevel
+func (loglevel *Loglevel) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var loglevelString string
+	err := unmarshal(&loglevelString)
+	if err != nil {
+		return err
+	}
+
+	loglevelString = strings.ToLower(loglevelString)
+	switch loglevelString {
+	case "error", "warn", "info", "debug":
+	default:
+		return fmt.Errorf("Invalid loglevel %s Must be one of [error, warn, info, debug]", loglevelString)
+	}
+
+	*loglevel = Loglevel(loglevelString)
+	return nil
+}
+
+// Storage defines the configuration for registry object storage
+type Storage map[string]Parameters
+
+// Type returns the storage driver type, such as filesystem or s3
+func (storage Storage) Type() string {
+	// Return only key in this map
+	for k := range storage {
+		return k
+	}
+	return ""
+}
+
+// Parameters returns the Parameters map for a Storage configuration
+func (storage Storage) Parameters() Parameters {
+	return storage[storage.Type()]
+}
+
+// setParameter changes the parameter at the provided key to the new value
+func (storage Storage) setParameter(key, value string) {
+	storage[storage.Type()][key] = value
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface
+// Unmarshals a single item map into a Storage or a string into a Storage type with no parameters
+func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var storageMap map[string]Parameters
+	err := unmarshal(&storageMap)
+	if err == nil {
+		if len(storageMap) > 1 {
+			types := make([]string, 0, len(storageMap))
+			for k := range storageMap {
+				types = append(types, k)
+			}
+			return fmt.Errorf("Must provide exactly one storage type. Provided: %v", types)
+		}
+		*storage = storageMap
+		return nil
+	}
+
+	var storageType string
+	err = unmarshal(&storageType)
+	if err == nil {
+		*storage = Storage{storageType: Parameters{}}
+		return nil
+	}
+
+	return err
+}
+
+// MarshalYAML implements the yaml.Marshaler interface
+func (storage Storage) MarshalYAML() (interface{}, error) {
+	if storage.Parameters == nil {
+		return storage.Type, nil
+	}
+	return map[string]Parameters(storage), nil
+}
+
+// Parameters defines a key-value parameters mapping
+type Parameters map[string]string
+
+// Parse parses an input configuration yaml document into a Configuration struct
+// This should generally be capable of handling old configuration format versions
+//
+// Environment variables may be used to override configuration parameters other than version,
+// following the scheme below:
+// Configuration.Abc may be replaced by the value of REGISTRY_ABC,
+// Configuration.Abc.Xyz may be replaced by the value of REGISTRY_ABC_XYZ, and so forth
+func Parse(in []byte) (*Configuration, error) {
+	var untypedConfig struct {
+		Version Version
+	}
+	var config *Configuration
+
+	err := yaml.Unmarshal(in, &untypedConfig)
+	if err != nil {
+		return nil, err
+	}
+	if untypedConfig.Version == "" {
+		return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
+	}
+
+	// Parse the remainder of the configuration depending on the provided version
+	switch untypedConfig.Version {
+	case "0.1":
+		config, err = parseV_0_1_Registry(in)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
+	}
+
+	return config, nil
+}
+
+// parseV_0_1_Registry parses a registry Configuration for Version 0.1
+func parseV_0_1_Registry(in []byte) (*Configuration, error) {
+	envMap := getEnvMap()
+
+	var config v_0_1_Configuration
+	err := yaml.Unmarshal(in, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	// Override config.Loglevel if environment variable is provided
+	if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
+		var newLoglevel Loglevel
+		err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
+		if err != nil {
+			return nil, err
+		}
+		config.Loglevel = newLoglevel
+	}
+
+	// Override config.Storage if environment variable is provided
+	if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
+		if storageType != config.Storage.Type() {
+			// Reset the storage parameters because we're using a different storage type
+			config.Storage = Storage{storageType: Parameters{}}
+		}
+	}
+
+	if config.Storage.Type() == "" {
+		return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
+	}
+
+	// Override storage parameters with all environment variables of the format:
+	// REGISTRY_STORAGE_<storage driver type>_<parameter name>
+	storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range envMap {
+		if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
+			config.Storage.setParameter(strings.ToLower(submatches[1]), v)
+		}
+	}
+
+	return (*Configuration)(&config), nil
+}
+
+// getEnvMap reads the current environment variables and converts these into a key/value map
+// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
+// environment variables and explicitly empty ones
+func getEnvMap() map[string]string {
+	envMap := make(map[string]string)
+	for _, env := range os.Environ() {
+		envParts := strings.SplitN(env, "=", 2)
+		envMap[envParts[0]] = envParts[1]
+	}
+	return envMap
+}

configuration/configuration_test.go

@@ -0,0 +1,195 @@
+package configuration
+
+import (
+	"os"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner
+func Test(t *testing.T) { TestingT(t) }
+
+// configStruct is a canonical example configuration, which should map to configYamlV_0_1
+var configStruct = Configuration{
+	Version:  "0.1",
+	Loglevel: "info",
+	Storage: Storage{
+		"s3": Parameters{
+			"region":    "us-east-1",
+			"bucket":    "my-bucket",
+			"rootpath":  "/registry",
+			"encrypt":   "true",
+			"secure":    "false",
+			"accesskey": "SAMPLEACCESSKEY",
+			"secretkey": "SUPERSECRET",
+			"host":      "",
+			"port":      "",
+		},
+	},
+}
+
+// configYamlV_0_1 is a Version 0.1 yaml document representing configStruct
+var configYamlV_0_1 = `
+version: 0.1
+loglevel: info
+storage:
+  s3:
+    region: us-east-1
+    bucket: my-bucket
+    rootpath: /registry
+    encrypt: true
+    secure: false
+    accesskey: SAMPLEACCESSKEY
+    secretkey: SUPERSECRET
+    host: ~
+    port: ~
+`
+
+// inmemoryConfigYamlV_0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
+// no parameters
+var inmemoryConfigYamlV_0_1 = `
+version: 0.1
+loglevel: info
+storage: inmemory
+`
+
+type ConfigSuite struct {
+	expectedConfig *Configuration
+}
+
+var _ = Suite(new(ConfigSuite))
+
+func (suite *ConfigSuite) SetUpTest(c *C) {
+	os.Clearenv()
+	suite.expectedConfig = copyConfig(configStruct)
+}
+
+// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
+// changing any parameters
+func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
+	configBytes, err := yaml.Marshal(suite.expectedConfig)
+	c.Assert(err, IsNil)
+	config, err := Parse(configBytes)
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseSimple validates that configYamlV_0_1 can be parsed into a struct matching configStruct
+func (suite *ConfigSuite) TestParseSimple(c *C) {
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseInmemory validates that configuration yaml with storage provided as a string can be
+// parsed into a Configuration struct with no storage parameters
+func (suite *ConfigSuite) TestParseInmemory(c *C) {
+	suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
+
+	config, err := Parse([]byte(inmemoryConfigYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithSameEnvStorage validates that providing environment variables that match the given
+// storage type and parameters will not alter the parsed Configuration struct
+func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
+	os.Setenv("REGISTRY_STORAGE", "s3")
+	os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithDifferentEnvStorageParams validates that providing environment variables that change
+// and add to the given storage parameters will change and add parameters to the parsed
+// Configuration struct
+func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams(c *C) {
+	suite.expectedConfig.Storage.setParameter("region", "us-west-1")
+	suite.expectedConfig.Storage.setParameter("secure", "true")
+	suite.expectedConfig.Storage.setParameter("newparam", "some Value")
+
+	os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-west-1")
+	os.Setenv("REGISTRY_STORAGE_S3_SECURE", "true")
+	os.Setenv("REGISTRY_STORAGE_S3_NEWPARAM", "some Value")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithDifferentEnvStorageType validates that providing an environment variable that
+// changes the storage type will be reflected in the parsed Configuration struct
+func (suite *ConfigSuite) TestParseWithDifferentEnvStorageType(c *C) {
+	suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
+
+	os.Setenv("REGISTRY_STORAGE", "inmemory")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithDifferentEnvStorageTypeAndParams validates that providing an environment variable
+// that changes the storage type will be reflected in the parsed Configuration struct and that
+// environment storage parameters will also be included
+func (suite *ConfigSuite) TestParseWithDifferentEnvStorageTypeAndParams(c *C) {
+	suite.expectedConfig.Storage = Storage{"filesystem": Parameters{}}
+	suite.expectedConfig.Storage.setParameter("rootdirectory", "/tmp/testroot")
+
+	os.Setenv("REGISTRY_STORAGE", "filesystem")
+	os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithSameEnvLoglevel validates that providing an environment variable defining the log
+// level to the same as the one provided in the yaml will not change the parsed Configuration struct
+func (suite *ConfigSuite) TestParseWithSameEnvLoglevel(c *C) {
+	os.Setenv("REGISTRY_LOGLEVEL", "info")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithDifferentEnvLoglevel validates that providing an environment variable defining the
+// log level will override the value provided in the yaml document
+func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
+	suite.expectedConfig.Loglevel = "error"
+
+	os.Setenv("REGISTRY_LOGLEVEL", "error")
+
+	config, err := Parse([]byte(configYamlV_0_1))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseInvalidVersion validates that the parser will fail to parse a newer configuration
+// version than the CurrentVersion
+func (suite *ConfigSuite) TestParseInvalidVersion(c *C) {
+	suite.expectedConfig.Version = MajorMinorVersion(CurrentVersion.Major(), CurrentVersion.Minor()+1)
+	configBytes, err := yaml.Marshal(suite.expectedConfig)
+	c.Assert(err, IsNil)
+	_, err = Parse(configBytes)
+	c.Assert(err, NotNil)
+}
+
+func copyConfig(config Configuration) *Configuration {
+	configCopy := new(Configuration)
+
+	configCopy.Version = MajorMinorVersion(config.Version.Major(), config.Version.Minor())
+	configCopy.Loglevel = config.Loglevel
+	configCopy.Storage = Storage{config.Storage.Type(): Parameters{}}
+	for k, v := range config.Storage.Parameters() {
+		configCopy.Storage.setParameter(k, v)
+	}
+
+	return configCopy
+}