TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions
distribution/configuration/configuration_test.go
James Hewitt c40c4b289a
Enable configuration of index dependency validation 
Enable configuration options that can selectively disable validation
that dependencies exist within the registry before the image index
is uploaded.

This enables sparse indexes, where a registry holds a manifest index that
could be signed (so the digest must not change) but does not hold every
referenced image in the index. The use case for this is when a registry
mirror does not need to mirror all platforms, but does need to maintain
the digests of all manifests either because they are signed or because
they are pulled by digest.

The registry administrator can also select specific image architectures
that must exist in the registry, enabling a registry operator to select
only the platforms they care about and ensure all image indexes uploaded
to the registry are valid for those platforms.

Signed-off-by: James Hewitt <james.hewitt@uk.ibm.com>
2024-05-28 09:56:14 +01:00

592 lines
19 KiB
Go
Raw Blame History

package configuration
import (
"bytes"
"net/http"
"reflect"
"strings"
"testing"
"time"
"github.com/stretchr/testify/suite"
"gopkg.in/yaml.v2"
)
// configStruct is a canonical example configuration, which should map to configYamlV0_1
var configStruct = Configuration{
Version: "0.1",
Log: struct {
AccessLog struct {
Disabled bool `yaml:"disabled,omitempty"`
} `yaml:"accesslog,omitempty"`
Level Loglevel `yaml:"level,omitempty"`
Formatter string `yaml:"formatter,omitempty"`
Fields map[string]interface{} `yaml:"fields,omitempty"`
Hooks []LogHook `yaml:"hooks,omitempty"`
ReportCaller bool `yaml:"reportcaller,omitempty"`
}{
Level: "info",
Fields: map[string]interface{}{"environment": "test"},
},
Storage: Storage{
"somedriver": Parameters{
"string1": "string-value1",
"string2": "string-value2",
"bool1": true,
"bool2": false,
"nil1": nil,
"int1": 42,
"url1": "https://foo.example.com",
"path1": "/some-path",
},
"tag": Parameters{
"concurrencylimit": 10,
},
},
Auth: Auth{
"silly": Parameters{
"realm": "silly",
"service": "silly",
},
},
Notifications: Notifications{
Endpoints: []Endpoint{
{
Name: "endpoint-1",
URL: "http://example.com",
Headers: http.Header{
"Authorization": []string{"Bearer <example>"},
},
IgnoredMediaTypes: []string{"application/octet-stream"},
Ignore: Ignore{
MediaTypes: []string{"application/octet-stream"},
Actions: []string{"pull"},
},
},
},
},
Catalog: Catalog{
MaxEntries: 1000,
},
HTTP: struct {
Addr string `yaml:"addr,omitempty"`
Net string `yaml:"net,omitempty"`
Host string `yaml:"host,omitempty"`
Prefix string `yaml:"prefix,omitempty"`
Secret string `yaml:"secret,omitempty"`
RelativeURLs bool `yaml:"relativeurls,omitempty"`
DrainTimeout time.Duration `yaml:"draintimeout,omitempty"`
TLS struct {
Certificate string `yaml:"certificate,omitempty"`
Key string `yaml:"key,omitempty"`
ClientCAs []string `yaml:"clientcas,omitempty"`
MinimumTLS string `yaml:"minimumtls,omitempty"`
CipherSuites []string `yaml:"ciphersuites,omitempty"`
LetsEncrypt struct {
CacheFile string `yaml:"cachefile,omitempty"`
Email string `yaml:"email,omitempty"`
Hosts []string `yaml:"hosts,omitempty"`
DirectoryURL string `yaml:"directoryurl,omitempty"`
} `yaml:"letsencrypt,omitempty"`
} `yaml:"tls,omitempty"`
Headers http.Header `yaml:"headers,omitempty"`
Debug struct {
Addr string `yaml:"addr,omitempty"`
Prometheus struct {
Enabled bool `yaml:"enabled,omitempty"`
Path string `yaml:"path,omitempty"`
} `yaml:"prometheus,omitempty"`
} `yaml:"debug,omitempty"`
HTTP2 struct {
Disabled bool `yaml:"disabled,omitempty"`
} `yaml:"http2,omitempty"`
H2C struct {
Enabled bool `yaml:"enabled,omitempty"`
} `yaml:"h2c,omitempty"`
}{
TLS: struct {
Certificate string `yaml:"certificate,omitempty"`
Key string `yaml:"key,omitempty"`
ClientCAs []string `yaml:"clientcas,omitempty"`
MinimumTLS string `yaml:"minimumtls,omitempty"`
CipherSuites []string `yaml:"ciphersuites,omitempty"`
LetsEncrypt struct {
CacheFile string `yaml:"cachefile,omitempty"`
Email string `yaml:"email,omitempty"`
Hosts []string `yaml:"hosts,omitempty"`
DirectoryURL string `yaml:"directoryurl,omitempty"`
} `yaml:"letsencrypt,omitempty"`
}{
ClientCAs: []string{"/path/to/ca.pem"},
},
Headers: http.Header{
"X-Content-Type-Options": []string{"nosniff"},
},
HTTP2: struct {
Disabled bool `yaml:"disabled,omitempty"`
}{
Disabled: false,
},
H2C: struct {
Enabled bool `yaml:"enabled,omitempty"`
}{
Enabled: true,
},
},
Redis: Redis{
Addr: "localhost:6379",
Username: "alice",
Password: "123456",
DB: 1,
Pool: struct {
MaxIdle int `yaml:"maxidle,omitempty"`
MaxActive int `yaml:"maxactive,omitempty"`
IdleTimeout time.Duration `yaml:"idletimeout,omitempty"`
}{
MaxIdle: 16,
MaxActive: 64,
IdleTimeout: time.Second * 300,
},
DialTimeout: time.Millisecond * 10,
ReadTimeout: time.Millisecond * 10,
WriteTimeout: time.Millisecond * 10,
},
Validation: Validation{
Manifests: ValidationManifests{
Indexes: ValidationIndexes{
Platforms: "none",
},
},
},
}
// configYamlV0_1 is a Version 0.1 yaml document representing configStruct
const configYamlV0_1 = `
version: 0.1
log:
level: info
fields:
environment: test
storage:
somedriver:
string1: string-value1
string2: string-value2
bool1: true
bool2: false
nil1: ~
int1: 42
url1: "https://foo.example.com"
path1: "/some-path"
tag:
concurrencylimit: 10
auth:
silly:
realm: silly
service: silly
notifications:
endpoints:
- name: endpoint-1
url: http://example.com
headers:
Authorization: [Bearer <example>]
ignoredmediatypes:
- application/octet-stream
ignore:
mediatypes:
- application/octet-stream
actions:
- pull
http:
clientcas:
- /path/to/ca.pem
headers:
X-Content-Type-Options: [nosniff]
redis:
addr: localhost:6379
username: alice
password: 123456
db: 1
pool:
maxidle: 16
maxactive: 64
idletimeout: 300s
dialtimeout: 10ms
readtimeout: 10ms
writetimeout: 10ms
validation:
manifests:
indexes:
platforms: none
`
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
// storage driver with no parameters
const inmemoryConfigYamlV0_1 = `
version: 0.1
log:
level: info
storage: inmemory
auth:
silly:
realm: silly
service: silly
notifications:
endpoints:
- name: endpoint-1
url: http://example.com
headers:
Authorization: [Bearer <example>]
ignoredmediatypes:
- application/octet-stream
ignore:
mediatypes:
- application/octet-stream
actions:
- pull
http:
headers:
X-Content-Type-Options: [nosniff]
validation:
manifests:
indexes:
platforms: none
`
type ConfigSuite struct {
suite.Suite
expectedConfig *Configuration
}
func TestConfigSuite(t *testing.T) {
suite.Run(t, new(ConfigSuite))
}
func (suite *ConfigSuite) SetupTest() {
suite.expectedConfig = copyConfig(configStruct)
}
// TestMarshalRoundtrip validates that configStruct can be marshaled and
// unmarshaled without changing any parameters
func (suite *ConfigSuite) TestMarshalRoundtrip() {
configBytes, err := yaml.Marshal(suite.expectedConfig)
suite.Require().NoError(err)
config, err := Parse(bytes.NewReader(configBytes))
suite.T().Log(string(configBytes))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct
// matching configStruct
func (suite *ConfigSuite) TestParseSimple() {
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseInmemory validates that configuration yaml with storage provided as
// a string can be parsed into a Configuration struct with no storage parameters
func (suite *ConfigSuite) TestParseInmemory() {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
suite.expectedConfig.Log.Fields = nil
suite.expectedConfig.Redis = Redis{}
config, err := Parse(bytes.NewReader([]byte(inmemoryConfigYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseIncomplete validates that an incomplete yaml configuration cannot
// be parsed without providing environment variables to fill in the missing
// components.
func (suite *ConfigSuite) TestParseIncomplete() {
incompleteConfigYaml := "version: 0.1"
_, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
suite.Require().Error(err)
suite.expectedConfig.Log.Fields = nil
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}}
suite.expectedConfig.Auth = Auth{"silly": Parameters{"realm": "silly"}}
suite.expectedConfig.Notifications = Notifications{}
suite.expectedConfig.HTTP.Headers = nil
suite.expectedConfig.Redis = Redis{}
suite.expectedConfig.Validation.Manifests.Indexes.Platforms = ""
// Note: this also tests that REGISTRY_STORAGE and
// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
suite.T().Setenv("REGISTRY_STORAGE", "filesystem")
suite.T().Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
suite.T().Setenv("REGISTRY_AUTH", "silly")
suite.T().Setenv("REGISTRY_AUTH_SILLY_REALM", "silly")
config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithSameEnvStorage validates that providing environment variables
// that match the given storage type will only include environment-defined
// parameters and remove yaml-defined parameters
func (suite *ConfigSuite) TestParseWithSameEnvStorage() {
suite.expectedConfig.Storage = Storage{"somedriver": Parameters{"region": "us-east-1"}}
suite.T().Setenv("REGISTRY_STORAGE", "somedriver")
suite.T().Setenv("REGISTRY_STORAGE_SOMEDRIVER_REGION", "us-east-1")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithDifferentEnvStorageParams validates that providing environment variables that change
// and add to the given storage parameters will change and add parameters to the parsed
// Configuration struct
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams() {
suite.expectedConfig.Storage.setParameter("string1", "us-west-1")
suite.expectedConfig.Storage.setParameter("bool1", true)
suite.expectedConfig.Storage.setParameter("newparam", "some Value")
suite.T().Setenv("REGISTRY_STORAGE_SOMEDRIVER_STRING1", "us-west-1")
suite.T().Setenv("REGISTRY_STORAGE_SOMEDRIVER_BOOL1", "true")
suite.T().Setenv("REGISTRY_STORAGE_SOMEDRIVER_NEWPARAM", "some Value")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithDifferentEnvStorageType validates that providing an environment variable that
// changes the storage type will be reflected in the parsed Configuration struct
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageType() {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
suite.T().Setenv("REGISTRY_STORAGE", "inmemory")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithDifferentEnvStorageTypeAndParams validates that providing an environment variable
// that changes the storage type will be reflected in the parsed Configuration struct and that
// environment storage parameters will also be included
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageTypeAndParams() {
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{}}
suite.expectedConfig.Storage.setParameter("rootdirectory", "/tmp/testroot")
suite.T().Setenv("REGISTRY_STORAGE", "filesystem")
suite.T().Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithSameEnvLoglevel validates that providing an environment variable defining the log
// level to the same as the one provided in the yaml will not change the parsed Configuration struct
func (suite *ConfigSuite) TestParseWithSameEnvLoglevel() {
suite.T().Setenv("REGISTRY_LOGLEVEL", "info")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseWithDifferentEnvLoglevel validates that providing an environment variable defining the
// log level will override the value provided in the yaml document
func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel() {
suite.expectedConfig.Log.Level = "error"
suite.T().Setenv("REGISTRY_LOG_LEVEL", "error")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseInvalidLoglevel validates that the parser will fail to parse a
// configuration if the loglevel is malformed
func (suite *ConfigSuite) TestParseInvalidLoglevel() {
invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory"
_, err := Parse(bytes.NewReader([]byte(invalidConfigYaml)))
suite.Require().Error(err)
suite.T().Setenv("REGISTRY_LOGLEVEL", "derp")
_, err = Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().Error(err)
}
// TestParseInvalidVersion validates that the parser will fail to parse a newer configuration
// version than the CurrentVersion
func (suite *ConfigSuite) TestParseInvalidVersion() {
suite.expectedConfig.Version = MajorMinorVersion(CurrentVersion.Major(), CurrentVersion.Minor()+1)
configBytes, err := yaml.Marshal(suite.expectedConfig)
suite.Require().NoError(err)
_, err = Parse(bytes.NewReader(configBytes))
suite.Require().Error(err)
}
// TestParseExtraneousVars validates that environment variables referring to
// nonexistent variables don't cause side effects.
func (suite *ConfigSuite) TestParseExtraneousVars() {
// Environment variables which shouldn't set config items
suite.T().Setenv("REGISTRY_DUCKS", "quack")
suite.T().Setenv("REGISTRY_REPORTING_ASDF", "ghjk")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseEnvVarImplicitMaps validates that environment variables can set
// values in maps that don't already exist.
func (suite *ConfigSuite) TestParseEnvVarImplicitMaps() {
readonly := make(map[string]interface{})
readonly["enabled"] = true
maintenance := make(map[string]interface{})
maintenance["readonly"] = readonly
suite.expectedConfig.Storage["maintenance"] = maintenance
suite.T().Setenv("REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED", "true")
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
suite.Require().Equal(suite.expectedConfig, config)
}
// TestParseEnvWrongTypeMap validates that incorrectly attempting to unmarshal a
// string over existing map fails.
func (suite *ConfigSuite) TestParseEnvWrongTypeMap() {
suite.T().Setenv("REGISTRY_STORAGE_SOMEDRIVER", "somestring")
_, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().Error(err)
}
// TestParseEnvWrongTypeStruct validates that incorrectly attempting to
// unmarshal a string into a struct fails.
func (suite *ConfigSuite) TestParseEnvWrongTypeStruct() {
suite.T().Setenv("REGISTRY_STORAGE_LOG", "somestring")
_, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().Error(err)
}
// TestParseEnvWrongTypeSlice validates that incorrectly attempting to
// unmarshal a string into a slice fails.
func (suite *ConfigSuite) TestParseEnvWrongTypeSlice() {
suite.T().Setenv("REGISTRY_LOG_HOOKS", "somestring")
_, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().Error(err)
}
// TestParseEnvMany tests several environment variable overrides.
// The result is not checked - the goal of this test is to detect panics
// from misuse of reflection.
func (suite *ConfigSuite) TestParseEnvMany() {
suite.T().Setenv("REGISTRY_VERSION", "0.1")
suite.T().Setenv("REGISTRY_LOG_LEVEL", "debug")
suite.T().Setenv("REGISTRY_LOG_FORMATTER", "json")
suite.T().Setenv("REGISTRY_LOG_HOOKS", "json")
suite.T().Setenv("REGISTRY_LOG_FIELDS", "abc: xyz")
suite.T().Setenv("REGISTRY_LOG_HOOKS", "- type: asdf")
suite.T().Setenv("REGISTRY_LOGLEVEL", "debug")
suite.T().Setenv("REGISTRY_STORAGE", "somedriver")
suite.T().Setenv("REGISTRY_AUTH_PARAMS", "param1: value1")
suite.T().Setenv("REGISTRY_AUTH_PARAMS_VALUE2", "value2")
suite.T().Setenv("REGISTRY_AUTH_PARAMS_VALUE2", "value2")
_, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
suite.Require().NoError(err)
}
func checkStructs(tt *testing.T, t reflect.Type, structsChecked map[string]struct{}) {
tt.Helper()
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Map || t.Kind() == reflect.Slice {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return
}
if _, present := structsChecked[t.String()]; present {
// Already checked this type
return
}
structsChecked[t.String()] = struct{}{}
byUpperCase := make(map[string]int)
for i := 0; i < t.NumField(); i++ {
sf := t.Field(i)
// Check that the yaml tag does not contain an _.
yamlTag := sf.Tag.Get("yaml")
if strings.Contains(yamlTag, "_") {
tt.Fatalf("yaml field name includes _ character: %s", yamlTag)
}
upper := strings.ToUpper(sf.Name)
if _, present := byUpperCase[upper]; present {
tt.Fatalf("field name collision in configuration object: %s", sf.Name)
}
byUpperCase[upper] = i
checkStructs(tt, sf.Type, structsChecked)
}
}
// TestValidateConfigStruct makes sure that the config struct has no members
// with yaml tags that would be ambiguous to the environment variable parser.
func (suite *ConfigSuite) TestValidateConfigStruct() {
structsChecked := make(map[string]struct{})
checkStructs(suite.T(), reflect.TypeOf(Configuration{}), structsChecked)
}
func copyConfig(config Configuration) *Configuration {
configCopy := new(Configuration)
configCopy.Version = MajorMinorVersion(config.Version.Major(), config.Version.Minor())
configCopy.Loglevel = config.Loglevel
configCopy.Log = config.Log
configCopy.Catalog = config.Catalog
configCopy.Log.Fields = make(map[string]interface{}, len(config.Log.Fields))
for k, v := range config.Log.Fields {
configCopy.Log.Fields[k] = v
}
configCopy.Storage = Storage{config.Storage.Type(): Parameters{}}
for k, v := range config.Storage.Parameters() {
configCopy.Storage.setParameter(k, v)
}
for k, v := range config.Storage.TagParameters() {
configCopy.Storage.setTagParameter(k, v)
}
configCopy.Auth = Auth{config.Auth.Type(): Parameters{}}
for k, v := range config.Auth.Parameters() {
configCopy.Auth.setParameter(k, v)
}
configCopy.Notifications = Notifications{Endpoints: []Endpoint{}}
configCopy.Notifications.Endpoints = append(configCopy.Notifications.Endpoints, config.Notifications.Endpoints...)
configCopy.HTTP.Headers = make(http.Header)
for k, v := range config.HTTP.Headers {
configCopy.HTTP.Headers[k] = v
}
configCopy.Redis = config.Redis
configCopy.Validation = Validation{
Enabled: config.Validation.Enabled,
Disabled: config.Validation.Disabled,
Manifests: config.Validation.Manifests,
}
return configCopy
}
Reference in a new issue View git blame Copy permalink