forked from TrueCloudLab/distribution
Genericizes the yaml+environment versioned configuration parser
Registry configuration parsing uses the new parser with a single version declaration and an environment prefix of "REGISTRY"
This commit is contained in:
Brian Bland
parent
454d4e918a
commit
f9b119974d
4 changed files with 274 additions and 149 deletions
|
|
@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir
|
|||
For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
|
||||
|
||||
### Notes
|
||||
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
|
||||
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration.
|
||||
|
||||
By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.
|
||||
|
|
|
|||
|
|
@ -4,12 +4,8 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/BrianBland/yaml.v2"
|
||||
)
|
||||
|
||||
// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
|
||||
|
|
@ -39,40 +35,6 @@ type Configuration struct {
|
|||
// This is currently aliased to Configuration, as it is the current version
|
||||
type v0_1Configuration Configuration
|
||||
|
||||
// Version is a major/minor version pair of the form Major.Minor
|
||||
// Major version upgrades indicate structure or type changes
|
||||
// Minor version upgrades should be strictly additive
|
||||
type Version string
|
||||
|
||||
// MajorMinorVersion constructs a Version from its Major and Minor components
|
||||
func MajorMinorVersion(major, minor uint) Version {
|
||||
return Version(fmt.Sprintf("%d.%d", major, minor))
|
||||
}
|
||||
|
||||
func (version Version) major() (uint, error) {
|
||||
majorPart := strings.Split(string(version), ".")[0]
|
||||
major, err := strconv.ParseUint(majorPart, 10, 0)
|
||||
return uint(major), err
|
||||
}
|
||||
|
||||
// Major returns the major version portion of a Version
|
||||
func (version Version) Major() uint {
|
||||
major, _ := version.major()
|
||||
return major
|
||||
}
|
||||
|
||||
func (version Version) minor() (uint, error) {
|
||||
minorPart := strings.Split(string(version), ".")[1]
|
||||
minor, err := strconv.ParseUint(minorPart, 10, 0)
|
||||
return uint(minor), err
|
||||
}
|
||||
|
||||
// Minor returns the minor version portion of a Version
|
||||
func (version Version) Minor() uint {
|
||||
minor, _ := version.minor()
|
||||
return minor
|
||||
}
|
||||
|
||||
// UnmarshalYAML implements the yaml.Unmarshaler interface
|
||||
// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
|
||||
func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
|
|
@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
var untypedConfig struct {
|
||||
Version Version
|
||||
}
|
||||
var config *Configuration
|
||||
p := NewParser("registry", []VersionedParseInfo{
|
||||
{
|
||||
Version: MajorMinorVersion(0, 1),
|
||||
ParseAs: reflect.TypeOf(v0_1Configuration{}),
|
||||
ConversionFunc: func(c interface{}) (interface{}, error) {
|
||||
if v0_1, ok := c.(*v0_1Configuration); ok {
|
||||
if v0_1.Loglevel == Loglevel("") {
|
||||
v0_1.Loglevel = Loglevel("info")
|
||||
}
|
||||
if v0_1.Storage.Type() == "" {
|
||||
return nil, fmt.Errorf("No storage configuration provided")
|
||||
}
|
||||
return (*Configuration)(v0_1), nil
|
||||
}
|
||||
return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c)
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if err := yaml.Unmarshal(in, &untypedConfig); err != nil {
|
||||
config := new(Configuration)
|
||||
err = p.Parse(in, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if untypedConfig.Version == "" {
|
||||
return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
|
||||
}
|
||||
|
||||
// Parse the remainder of the configuration depending on the provided version
|
||||
switch untypedConfig.Version {
|
||||
case "0.1":
|
||||
config, err = parseV0_1Registry(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// parseV0_1Registry parses a registry Configuration for Version 0.1
|
||||
func parseV0_1Registry(in []byte) (*Configuration, error) {
|
||||
envMap := getEnvMap()
|
||||
|
||||
var config v0_1Configuration
|
||||
err := yaml.Unmarshal(in, &config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Override config.Loglevel if environment variable is provided
|
||||
if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
|
||||
var newLoglevel Loglevel
|
||||
err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.Loglevel = newLoglevel
|
||||
}
|
||||
|
||||
// Override config.Storage if environment variable is provided
|
||||
if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
|
||||
if storageType != config.Storage.Type() {
|
||||
// Reset the storage parameters because we're using a different storage type
|
||||
config.Storage = Storage{storageType: Parameters{}}
|
||||
}
|
||||
}
|
||||
|
||||
if config.Storage.Type() == "" {
|
||||
return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
|
||||
}
|
||||
|
||||
// Override storage parameters with all environment variables of the format:
|
||||
// REGISTRY_STORAGE_<storage driver type>_<parameter name>
|
||||
storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range envMap {
|
||||
if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
|
||||
config.Storage.setParameter(strings.ToLower(submatches[1]), v)
|
||||
}
|
||||
}
|
||||
|
||||
if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok {
|
||||
config.Reporting.Bugsnag.APIKey = bugsnagAPIKey
|
||||
}
|
||||
if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok {
|
||||
config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage
|
||||
}
|
||||
if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok {
|
||||
config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint
|
||||
}
|
||||
|
||||
if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok {
|
||||
config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey
|
||||
}
|
||||
if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok {
|
||||
config.Reporting.NewRelic.Name = newRelicName
|
||||
}
|
||||
|
||||
if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok {
|
||||
config.HTTP.Addr = httpAddr
|
||||
}
|
||||
|
||||
return (*Configuration)(&config), nil
|
||||
}
|
||||
|
||||
// getEnvMap reads the current environment variables and converts these into a key/value map
|
||||
// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
|
||||
// environment variables and explicitly empty ones
|
||||
func getEnvMap() map[string]string {
|
||||
envMap := make(map[string]string)
|
||||
for _, env := range os.Environ() {
|
||||
envParts := strings.SplitN(env, "=", 2)
|
||||
envMap[envParts[0]] = envParts[1]
|
||||
}
|
||||
return envMap
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ reporting:
|
|||
apikey: BugsnagApiKey
|
||||
`
|
||||
|
||||
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
|
||||
// no parameters
|
||||
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
|
||||
// storage driver with no parameters
|
||||
var inmemoryConfigYamlV0_1 = `
|
||||
version: 0.1
|
||||
loglevel: info
|
||||
|
|
@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) {
|
|||
suite.expectedConfig = copyConfig(configStruct)
|
||||
}
|
||||
|
||||
// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
|
||||
// changing any parameters
|
||||
// TestMarshalRoundtrip validates that configStruct can be marshaled and
|
||||
// unmarshaled without changing any parameters
|
||||
func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
|
||||
configBytes, err := yaml.Marshal(suite.expectedConfig)
|
||||
c.Assert(err, IsNil)
|
||||
|
|
@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
|
|||
c.Assert(config, DeepEquals, suite.expectedConfig)
|
||||
}
|
||||
|
||||
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct
|
||||
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct
|
||||
// matching configStruct
|
||||
func (suite *ConfigSuite) TestParseSimple(c *C) {
|
||||
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(config, DeepEquals, suite.expectedConfig)
|
||||
}
|
||||
|
||||
// TestParseInmemory validates that configuration yaml with storage provided as a string can be
|
||||
// parsed into a Configuration struct with no storage parameters
|
||||
// TestParseInmemory validates that configuration yaml with storage provided as
|
||||
// a string can be parsed into a Configuration struct with no storage parameters
|
||||
func (suite *ConfigSuite) TestParseInmemory(c *C) {
|
||||
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
|
||||
suite.expectedConfig.Reporting = Reporting{}
|
||||
|
|
@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) {
|
|||
c.Assert(config, DeepEquals, suite.expectedConfig)
|
||||
}
|
||||
|
||||
// TestParseWithSameEnvStorage validates that providing environment variables that match the given
|
||||
// storage type and parameters will not alter the parsed Configuration struct
|
||||
// TestParseIncomplete validates that an incomplete yaml configuration cannot
|
||||
// be parsed without providing environment variables to fill in the missing
|
||||
// components.
|
||||
func (suite *ConfigSuite) TestParseIncomplete(c *C) {
|
||||
incompleteConfigYaml := "version: 0.1"
|
||||
_, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
|
||||
c.Assert(err, NotNil)
|
||||
|
||||
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}}
|
||||
suite.expectedConfig.Reporting = Reporting{}
|
||||
|
||||
os.Setenv("REGISTRY_STORAGE", "filesystem")
|
||||
os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
|
||||
|
||||
config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(config, DeepEquals, suite.expectedConfig)
|
||||
}
|
||||
|
||||
// TestParseWithSameEnvStorage validates that providing environment variables
|
||||
// that match the given storage type will only include environment-defined
|
||||
// parameters and remove yaml-defined parameters
|
||||
func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
|
||||
suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}}
|
||||
|
||||
os.Setenv("REGISTRY_STORAGE", "s3")
|
||||
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
|
||||
|
||||
|
|
@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
|
|||
c.Assert(config, DeepEquals, suite.expectedConfig)
|
||||
}
|
||||
|
||||
// TestParseInvalidLoglevel validates that the parser will fail to parse a
|
||||
// configuration if the loglevel is malformed
|
||||
func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) {
|
||||
invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory"
|
||||
_, err := Parse(bytes.NewReader([]byte(invalidConfigYaml)))
|
||||
c.Assert(err, NotNil)
|
||||
|
||||
os.Setenv("REGISTRY_LOGLEVEL", "derp")
|
||||
|
||||
_, err = Parse(bytes.NewReader([]byte(configYamlV0_1)))
|
||||
c.Assert(err, NotNil)
|
||||
|
||||
}
|
||||
|
||||
// TestParseWithDifferentEnvReporting validates that environment variables
|
||||
// properly override reporting parameters
|
||||
func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) {
|
||||
suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey"
|
||||
suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080"
|
||||
|
|
|
|||
203
configuration/parser.go
Normal file
203
configuration/parser.go
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/BrianBland/yaml.v2"
|
||||
)
|
||||
|
||||
// Version is a major/minor version pair of the form Major.Minor
|
||||
// Major version upgrades indicate structure or type changes
|
||||
// Minor version upgrades should be strictly additive
|
||||
type Version string
|
||||
|
||||
// MajorMinorVersion constructs a Version from its Major and Minor components
|
||||
func MajorMinorVersion(major, minor uint) Version {
|
||||
return Version(fmt.Sprintf("%d.%d", major, minor))
|
||||
}
|
||||
|
||||
func (version Version) major() (uint, error) {
|
||||
majorPart := strings.Split(string(version), ".")[0]
|
||||
major, err := strconv.ParseUint(majorPart, 10, 0)
|
||||
return uint(major), err
|
||||
}
|
||||
|
||||
// Major returns the major version portion of a Version
|
||||
func (version Version) Major() uint {
|
||||
major, _ := version.major()
|
||||
return major
|
||||
}
|
||||
|
||||
func (version Version) minor() (uint, error) {
|
||||
minorPart := strings.Split(string(version), ".")[1]
|
||||
minor, err := strconv.ParseUint(minorPart, 10, 0)
|
||||
return uint(minor), err
|
||||
}
|
||||
|
||||
// Minor returns the minor version portion of a Version
|
||||
func (version Version) Minor() uint {
|
||||
minor, _ := version.minor()
|
||||
return minor
|
||||
}
|
||||
|
||||
// VersionedParseInfo defines how a specific version of a configuration should
|
||||
// be parsed into the current version
|
||||
type VersionedParseInfo struct {
|
||||
// Version is the version which this parsing information relates to
|
||||
Version Version
|
||||
// ParseAs defines the type which a configuration file of this version
|
||||
// should be parsed into
|
||||
ParseAs reflect.Type
|
||||
// ConversionFunc defines a method for converting the parsed configuration
|
||||
// (of type ParseAs) into the current configuration version
|
||||
// Note: this method signature is very unclear with the absence of generics
|
||||
ConversionFunc func(interface{}) (interface{}, error)
|
||||
}
|
||||
|
||||
// Parser can be used to parse a configuration file and environment of a defined
|
||||
// version into a unified output structure
|
||||
type Parser struct {
|
||||
prefix string
|
||||
mapping map[Version]VersionedParseInfo
|
||||
env map[string]string
|
||||
}
|
||||
|
||||
// NewParser returns a *Parser with the given environment prefix which handles
|
||||
// versioned configurations which match the given parseInfos
|
||||
func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
|
||||
p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)}
|
||||
|
||||
for _, parseInfo := range parseInfos {
|
||||
p.mapping[parseInfo.Version] = parseInfo
|
||||
}
|
||||
|
||||
for _, env := range os.Environ() {
|
||||
envParts := strings.SplitN(env, "=", 2)
|
||||
p.env[envParts[0]] = envParts[1]
|
||||
}
|
||||
|
||||
return &p
|
||||
}
|
||||
|
||||
// Parse reads in the given []byte and environment and writes the resulting
|
||||
// configuration into the input v
|
||||
//
|
||||
// Environment variables may be used to override configuration parameters other
|
||||
// than version, following the scheme below:
|
||||
// v.Abc may be replaced by the value of PREFIX_ABC,
|
||||
// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth
|
||||
func (p *Parser) Parse(in []byte, v interface{}) error {
|
||||
var versionedStruct struct {
|
||||
Version Version
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(in, &versionedStruct); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parseInfo, ok := p.mapping[versionedStruct.Version]
|
||||
if !ok {
|
||||
return fmt.Errorf("Unsupported version: %q", versionedStruct.Version)
|
||||
}
|
||||
|
||||
parseAs := reflect.New(parseInfo.ParseAs)
|
||||
err := yaml.Unmarshal(in, parseAs.Interface())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = p.overwriteFields(parseAs, p.prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c, err := parseInfo.ConversionFunc(parseAs.Interface())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Parser) overwriteFields(v reflect.Value, prefix string) error {
|
||||
for v.Kind() == reflect.Ptr {
|
||||
v = reflect.Indirect(v)
|
||||
}
|
||||
switch v.Kind() {
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
sf := v.Type().Field(i)
|
||||
fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name)
|
||||
if e, ok := p.env[fieldPrefix]; ok {
|
||||
fieldVal := reflect.New(sf.Type)
|
||||
err := yaml.Unmarshal([]byte(e), fieldVal.Interface())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v.Field(i).Set(reflect.Indirect(fieldVal))
|
||||
}
|
||||
err := p.overwriteFields(v.Field(i), fieldPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case reflect.Map:
|
||||
p.overwriteMap(v, prefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Parser) overwriteMap(m reflect.Value, prefix string) error {
|
||||
switch m.Type().Elem().Kind() {
|
||||
case reflect.Struct:
|
||||
for _, k := range m.MapKeys() {
|
||||
err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for key, val := range p.env {
|
||||
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
|
||||
mapValue := reflect.New(m.Type().Elem())
|
||||
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
|
||||
}
|
||||
}
|
||||
case reflect.Map:
|
||||
for _, k := range m.MapKeys() {
|
||||
err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for key, val := range p.env {
|
||||
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
|
||||
mapValue := reflect.New(m.Type().Elem())
|
||||
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in a new issue