TrueCloudLab/distribution
13
0
Fork 3
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #861 from BrianBland/ng-configuration

Browse source 
Genericizes the yaml+environment versioned configuration parser
This commit is contained in:
Stephen Day 2014-12-18 10:44:25 -08:00
commit e20c6ea4fb
7 changed files with 288 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
configuration/README.md
View file

@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir
For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
### Notes
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration.
By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.

165
configuration/configuration.go
View file

@ -4,12 +4,8 @@ import (
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"strconv"
"reflect"
"strings"
"gopkg.in/BrianBland/yaml.v2"
)
// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
@ -39,40 +35,6 @@ type Configuration struct {
// This is currently aliased to Configuration, as it is the current version
type v0_1Configuration Configuration
// Version is a major/minor version pair of the form Major.Minor
// Major version upgrades indicate structure or type changes
// Minor version upgrades should be strictly additive
type Version string
// MajorMinorVersion constructs a Version from its Major and Minor components
func MajorMinorVersion(major, minor uint) Version {
return Version(fmt.Sprintf("%d.%d", major, minor))
}
func (version Version) major() (uint, error) {
majorPart := strings.Split(string(version), ".")[0]
major, err := strconv.ParseUint(majorPart, 10, 0)
return uint(major), err
}
// Major returns the major version portion of a Version
func (version Version) Major() uint {
major, _ := version.major()
return major
}
func (version Version) minor() (uint, error) {
minorPart := strings.Split(string(version), ".")[1]
minor, err := strconv.ParseUint(minorPart, 10, 0)
return uint(minor), err
}
// Minor returns the minor version portion of a Version
func (version Version) Minor() uint {
minor, _ := version.minor()
return minor
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
@ -141,7 +103,7 @@ func (storage Storage) Parameters() Parameters {
}
// setParameter changes the parameter at the provided key to the new value
func (storage Storage) setParameter(key, value string) {
func (storage Storage) setParameter(key string, value interface{}) {
storage[storage.Type()][key] = value
}
@ -181,7 +143,7 @@ func (storage Storage) MarshalYAML() (interface{}, error) {
}
// Parameters defines a key-value parameters mapping
type Parameters map[string]string
type Parameters map[string]interface{}
// Reporting defines error reporting methods.
type Reporting struct {
@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) {
return nil, err
}
var untypedConfig struct {
Version Version
}
var config *Configuration
p := NewParser("registry", []VersionedParseInfo{
{
Version: MajorMinorVersion(0, 1),
ParseAs: reflect.TypeOf(v0_1Configuration{}),
ConversionFunc: func(c interface{}) (interface{}, error) {
if v0_1, ok := c.(*v0_1Configuration); ok {
if v0_1.Loglevel == Loglevel("") {
v0_1.Loglevel = Loglevel("info")
}
if v0_1.Storage.Type() == "" {
return nil, fmt.Errorf("No storage configuration provided")
}
return (*Configuration)(v0_1), nil
}
return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c)
},
},
})
if err := yaml.Unmarshal(in, &untypedConfig); err != nil {
config := new(Configuration)
err = p.Parse(in, config)
if err != nil {
return nil, err
}
if untypedConfig.Version == "" {
return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
}
// Parse the remainder of the configuration depending on the provided version
switch untypedConfig.Version {
case "0.1":
config, err = parseV0_1Registry(in)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
}
return config, nil
}
// parseV0_1Registry parses a registry Configuration for Version 0.1
func parseV0_1Registry(in []byte) (*Configuration, error) {
envMap := getEnvMap()
var config v0_1Configuration
err := yaml.Unmarshal(in, &config)
if err != nil {
return nil, err
}
// Override config.Loglevel if environment variable is provided
if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
var newLoglevel Loglevel
err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
if err != nil {
return nil, err
}
config.Loglevel = newLoglevel
}
// Override config.Storage if environment variable is provided
if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
if storageType != config.Storage.Type() {
// Reset the storage parameters because we're using a different storage type
config.Storage = Storage{storageType: Parameters{}}
}
}
if config.Storage.Type() == "" {
return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
}
// Override storage parameters with all environment variables of the format:
// REGISTRY_STORAGE_<storage driver type>_<parameter name>
storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
if err != nil {
return nil, err
}
for k, v := range envMap {
if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
config.Storage.setParameter(strings.ToLower(submatches[1]), v)
}
}
if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok {
config.Reporting.Bugsnag.APIKey = bugsnagAPIKey
}
if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok {
config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage
}
if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok {
config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint
}
if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok {
config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey
}
if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok {
config.Reporting.NewRelic.Name = newRelicName
}
if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok {
config.HTTP.Addr = httpAddr
}
return (*Configuration)(&config), nil
}
// getEnvMap reads the current environment variables and converts these into a key/value map
// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
// environment variables and explicitly empty ones
func getEnvMap() map[string]string {
envMap := make(map[string]string)
for _, env := range os.Environ() {
envParts := strings.SplitN(env, "=", 2)
envMap[envParts[0]] = envParts[1]
}
return envMap
}

69
configuration/configuration_test.go
View file

@ -21,12 +21,12 @@ var configStruct = Configuration{
"region": "us-east-1",
"bucket": "my-bucket",
"rootpath": "/registry",
"encrypt": "true",
"secure": "false",
"encrypt": true,
"secure": false,
"accesskey": "SAMPLEACCESSKEY",
"secretkey": "SUPERSECRET",
"host": "",
"port": "",
"host": nil,
"port": 42,
},
},
Reporting: Reporting{
@ -50,14 +50,14 @@ storage:
accesskey: SAMPLEACCESSKEY
secretkey: SUPERSECRET
host: ~
port: ~
port: 42
reporting:
bugsnag:
apikey: BugsnagApiKey
`
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
// no parameters
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
// storage driver with no parameters
var inmemoryConfigYamlV0_1 = `
version: 0.1
loglevel: info
@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) {
suite.expectedConfig = copyConfig(configStruct)
}
// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
// changing any parameters
// TestMarshalRoundtrip validates that configStruct can be marshaled and
// unmarshaled without changing any parameters
func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
configBytes, err := yaml.Marshal(suite.expectedConfig)
c.Assert(err, IsNil)
@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct
// matching configStruct
func (suite *ConfigSuite) TestParseSimple(c *C) {
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInmemory validates that configuration yaml with storage provided as a string can be
// parsed into a Configuration struct with no storage parameters
// TestParseInmemory validates that configuration yaml with storage provided as
// a string can be parsed into a Configuration struct with no storage parameters
func (suite *ConfigSuite) TestParseInmemory(c *C) {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
suite.expectedConfig.Reporting = Reporting{}
@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvStorage validates that providing environment variables that match the given
// storage type and parameters will not alter the parsed Configuration struct
// TestParseIncomplete validates that an incomplete yaml configuration cannot
// be parsed without providing environment variables to fill in the missing
// components.
func (suite *ConfigSuite) TestParseIncomplete(c *C) {
incompleteConfigYaml := "version: 0.1"
_, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
c.Assert(err, NotNil)
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}}
suite.expectedConfig.Reporting = Reporting{}
os.Setenv("REGISTRY_STORAGE", "filesystem")
os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvStorage validates that providing environment variables
// that match the given storage type will only include environment-defined
// parameters and remove yaml-defined parameters
func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}}
os.Setenv("REGISTRY_STORAGE", "s3")
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
@ -119,7 +142,7 @@ func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
// Configuration struct
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams(c *C) {
suite.expectedConfig.Storage.setParameter("region", "us-west-1")
suite.expectedConfig.Storage.setParameter("secure", "true")
suite.expectedConfig.Storage.setParameter("secure", true)
suite.expectedConfig.Storage.setParameter("newparam", "some Value")
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-west-1")
@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInvalidLoglevel validates that the parser will fail to parse a
// configuration if the loglevel is malformed
func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) {
invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory"
_, err := Parse(bytes.NewReader([]byte(invalidConfigYaml)))
c.Assert(err, NotNil)
os.Setenv("REGISTRY_LOGLEVEL", "derp")
_, err = Parse(bytes.NewReader([]byte(configYamlV0_1)))
c.Assert(err, NotNil)
}
// TestParseWithDifferentEnvReporting validates that environment variables
// properly override reporting parameters
func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) {
suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey"
suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080"

203
configuration/parser.go Normal file
View file

@ -0,0 +1,203 @@
package configuration
import (
"fmt"
"os"
"reflect"
"regexp"
"strconv"
"strings"
"gopkg.in/BrianBland/yaml.v2"
)
// Version is a major/minor version pair of the form Major.Minor
// Major version upgrades indicate structure or type changes
// Minor version upgrades should be strictly additive
type Version string
// MajorMinorVersion constructs a Version from its Major and Minor components
func MajorMinorVersion(major, minor uint) Version {
return Version(fmt.Sprintf("%d.%d", major, minor))
}
func (version Version) major() (uint, error) {
majorPart := strings.Split(string(version), ".")[0]
major, err := strconv.ParseUint(majorPart, 10, 0)
return uint(major), err
}
// Major returns the major version portion of a Version
func (version Version) Major() uint {
major, _ := version.major()
return major
}
func (version Version) minor() (uint, error) {
minorPart := strings.Split(string(version), ".")[1]
minor, err := strconv.ParseUint(minorPart, 10, 0)
return uint(minor), err
}
// Minor returns the minor version portion of a Version
func (version Version) Minor() uint {
minor, _ := version.minor()
return minor
}
// VersionedParseInfo defines how a specific version of a configuration should
// be parsed into the current version
type VersionedParseInfo struct {
// Version is the version which this parsing information relates to
Version Version
// ParseAs defines the type which a configuration file of this version
// should be parsed into
ParseAs reflect.Type
// ConversionFunc defines a method for converting the parsed configuration
// (of type ParseAs) into the current configuration version
// Note: this method signature is very unclear with the absence of generics
ConversionFunc func(interface{}) (interface{}, error)
}
// Parser can be used to parse a configuration file and environment of a defined
// version into a unified output structure
type Parser struct {
prefix string
mapping map[Version]VersionedParseInfo
env map[string]string
}
// NewParser returns a *Parser with the given environment prefix which handles
// versioned configurations which match the given parseInfos
func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)}
for _, parseInfo := range parseInfos {
p.mapping[parseInfo.Version] = parseInfo
}
for _, env := range os.Environ() {
envParts := strings.SplitN(env, "=", 2)
p.env[envParts[0]] = envParts[1]
}
return &p
}
// Parse reads in the given []byte and environment and writes the resulting
// configuration into the input v
//
// Environment variables may be used to override configuration parameters other
// than version, following the scheme below:
// v.Abc may be replaced by the value of PREFIX_ABC,
// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth
func (p *Parser) Parse(in []byte, v interface{}) error {
var versionedStruct struct {
Version Version
}
if err := yaml.Unmarshal(in, &versionedStruct); err != nil {
return err
}
parseInfo, ok := p.mapping[versionedStruct.Version]
if !ok {
return fmt.Errorf("Unsupported version: %q", versionedStruct.Version)
}
parseAs := reflect.New(parseInfo.ParseAs)
err := yaml.Unmarshal(in, parseAs.Interface())
if err != nil {
return err
}
err = p.overwriteFields(parseAs, p.prefix)
if err != nil {
return err
}
c, err := parseInfo.ConversionFunc(parseAs.Interface())
if err != nil {
return err
}
reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c)))
return nil
}
func (p *Parser) overwriteFields(v reflect.Value, prefix string) error {
for v.Kind() == reflect.Ptr {
v = reflect.Indirect(v)
}
switch v.Kind() {
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
sf := v.Type().Field(i)
fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name)
if e, ok := p.env[fieldPrefix]; ok {
fieldVal := reflect.New(sf.Type)
err := yaml.Unmarshal([]byte(e), fieldVal.Interface())
if err != nil {
return err
}
v.Field(i).Set(reflect.Indirect(fieldVal))
}
err := p.overwriteFields(v.Field(i), fieldPrefix)
if err != nil {
return err
}
}
case reflect.Map:
p.overwriteMap(v, prefix)
}
return nil
}
func (p *Parser) overwriteMap(m reflect.Value, prefix string) error {
switch m.Type().Elem().Kind() {
case reflect.Struct:
for _, k := range m.MapKeys() {
err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
if err != nil {
return err
}
}
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
if err != nil {
return err
}
for key, val := range p.env {
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
mapValue := reflect.New(m.Type().Elem())
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
if err != nil {
return err
}
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
}
}
case reflect.Map:
for _, k := range m.MapKeys() {
err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
if err != nil {
return err
}
}
default:
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
if err != nil {
return err
}
for key, val := range p.env {
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
mapValue := reflect.New(m.Type().Elem())
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
if err != nil {
return err
}
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
}
}
}
return nil
}

4
storagedriver/factory/factory.go
View file

@ -16,7 +16,7 @@ type StorageDriverFactory interface {
// Create returns a new storagedriver.StorageDriver with the given parameters
// Parameters will vary by driver and may be ignored
// Each parameter key must only consist of lowercase letters and numbers
Create(parameters map[string]string) (storagedriver.StorageDriver, error)
Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error)
}
// Register makes a storage driver available by the provided name.
@ -37,7 +37,7 @@ func Register(name string, factory StorageDriverFactory) {
// To run in-process, the StorageDriverFactory must first be registered with the given name
// If no in-process drivers are found with the given name, this attempts to create an IPC driver
// If no in-process or external drivers are found, an InvalidStorageDriverError is returned
func Create(name string, parameters map[string]string) (storagedriver.StorageDriver, error) {
func Create(name string, parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
driverFactory, ok := driverFactories[name]
if !ok {
return nil, InvalidStorageDriverError{name}

6
storagedriver/filesystem/driver.go
View file

@ -23,7 +23,7 @@ func init() {
// filesystemDriverFactory implements the factory.StorageDriverFactory interface
type filesystemDriverFactory struct{}
func (factory *filesystemDriverFactory) Create(parameters map[string]string) (storagedriver.StorageDriver, error) {
func (factory *filesystemDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
return FromParameters(parameters), nil
}
@ -36,12 +36,12 @@ type Driver struct {
// FromParameters constructs a new Driver with a given parameters map
// Optional Parameters:
// - rootdirectory
func FromParameters(parameters map[string]string) *Driver {
func FromParameters(parameters map[string]interface{}) *Driver {
var rootDirectory = defaultRootDirectory
if parameters != nil {
rootDir, ok := parameters["rootdirectory"]
if ok {
rootDirectory = rootDir
rootDirectory = fmt.Sprint(rootDir)
}
}
return New(rootDirectory)

2
storagedriver/inmemory/driver.go
View file

@ -21,7 +21,7 @@ func init() {
// inMemoryDriverFacotry implements the factory.StorageDriverFactory interface.
type inMemoryDriverFactory struct{}
func (factory *inMemoryDriverFactory) Create(parameters map[string]string) (storagedriver.StorageDriver, error) {
func (factory *inMemoryDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
return New(), nil
}