Add option to enable sparse indexes (#3536)

This commit is contained in:
Milos Gajdos 2024-05-28 10:15:02 +01:00 committed by GitHub
commit 37b83869a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 520 additions and 95 deletions

View file

@ -181,25 +181,7 @@ type Configuration struct {
Proxy Proxy `yaml:"proxy,omitempty"` Proxy Proxy `yaml:"proxy,omitempty"`
// Validation configures validation options for the registry. // Validation configures validation options for the registry.
Validation struct { Validation Validation `yaml:"validation,omitempty"`
// Enabled enables the other options in this section. This field is
// deprecated in favor of Disabled.
Enabled bool `yaml:"enabled,omitempty"`
// Disabled disables the other options in this section.
Disabled bool `yaml:"disabled,omitempty"`
// Manifests configures manifest validation.
Manifests struct {
// URLs configures validation for URLs in pushed manifests.
URLs struct {
// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
// that URLs in pushed manifests must match.
Allow []string `yaml:"allow,omitempty"`
// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
// that URLs in pushed manifests must not match.
Deny []string `yaml:"deny,omitempty"`
} `yaml:"urls,omitempty"`
} `yaml:"manifests,omitempty"`
} `yaml:"validation,omitempty"`
// Policy configures registry policy options. // Policy configures registry policy options.
Policy struct { Policy struct {
@ -366,6 +348,13 @@ type Health struct {
} `yaml:"storagedriver,omitempty"` } `yaml:"storagedriver,omitempty"`
} }
type Platform struct {
// Architecture is the architecture for this platform
Architecture string `yaml:"architecture,omitempty"`
// OS is the operating system for this platform
OS string `yaml:"os,omitempty"`
}
// v0_1Configuration is a Version 0.1 Configuration struct // v0_1Configuration is a Version 0.1 Configuration struct
// This is currently aliased to Configuration, as it is the current version // This is currently aliased to Configuration, as it is the current version
type v0_1Configuration Configuration type v0_1Configuration Configuration
@ -653,6 +642,62 @@ type Proxy struct {
TTL *time.Duration `yaml:"ttl,omitempty"` TTL *time.Duration `yaml:"ttl,omitempty"`
} }
type Validation struct {
// Enabled enables the other options in this section. This field is
// deprecated in favor of Disabled.
Enabled bool `yaml:"enabled,omitempty"`
// Disabled disables the other options in this section.
Disabled bool `yaml:"disabled,omitempty"`
// Manifests configures manifest validation.
Manifests ValidationManifests `yaml:"manifests,omitempty"`
}
type ValidationManifests struct {
// URLs configures validation for URLs in pushed manifests.
URLs struct {
// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
// that URLs in pushed manifests must match.
Allow []string `yaml:"allow,omitempty"`
// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
// that URLs in pushed manifests must not match.
Deny []string `yaml:"deny,omitempty"`
} `yaml:"urls,omitempty"`
// ImageIndexes configures validation of image indexes
Indexes ValidationIndexes `yaml:"indexes,omitempty"`
}
type ValidationIndexes struct {
// Platforms configures the validation applies to the platform images included in an image index
Platforms Platforms `yaml:"platforms"`
// PlatformList filters the set of platforms to validate for image existence.
PlatformList []Platform `yaml:"platformlist,omitempty"`
}
// Platforms configures the validation applies to the platform images included in an image index
// This can be all, none, or list
type Platforms string
// UnmarshalYAML implements the yaml.Umarshaler interface
// Unmarshals a string into a Platforms option, lowercasing the string and validating that it represents a
// valid option
func (platforms *Platforms) UnmarshalYAML(unmarshal func(interface{}) error) error {
var platformsString string
err := unmarshal(&platformsString)
if err != nil {
return err
}
platformsString = strings.ToLower(platformsString)
switch platformsString {
case "all", "none", "list":
default:
return fmt.Errorf("invalid platforms option %s Must be one of [all, none, list]", platformsString)
}
*platforms = Platforms(platformsString)
return nil
}
// Parse parses an input configuration yaml document into a Configuration struct // Parse parses an input configuration yaml document into a Configuration struct
// This should generally be capable of handling old configuration format versions // This should generally be capable of handling old configuration format versions
// //

View file

@ -151,6 +151,13 @@ var configStruct = Configuration{
ReadTimeout: time.Millisecond * 10, ReadTimeout: time.Millisecond * 10,
WriteTimeout: time.Millisecond * 10, WriteTimeout: time.Millisecond * 10,
}, },
Validation: Validation{
Manifests: ValidationManifests{
Indexes: ValidationIndexes{
Platforms: "none",
},
},
},
} }
// configYamlV0_1 is a Version 0.1 yaml document representing configStruct // configYamlV0_1 is a Version 0.1 yaml document representing configStruct
@ -206,6 +213,10 @@ redis:
dialtimeout: 10ms dialtimeout: 10ms
readtimeout: 10ms readtimeout: 10ms
writetimeout: 10ms writetimeout: 10ms
validation:
manifests:
indexes:
platforms: none
` `
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory // inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
@ -235,6 +246,10 @@ notifications:
http: http:
headers: headers:
X-Content-Type-Options: [nosniff] X-Content-Type-Options: [nosniff]
validation:
manifests:
indexes:
platforms: none
` `
type ConfigSuite struct { type ConfigSuite struct {
@ -295,6 +310,7 @@ func (suite *ConfigSuite) TestParseIncomplete() {
suite.expectedConfig.Notifications = Notifications{} suite.expectedConfig.Notifications = Notifications{}
suite.expectedConfig.HTTP.Headers = nil suite.expectedConfig.HTTP.Headers = nil
suite.expectedConfig.Redis = Redis{} suite.expectedConfig.Redis = Redis{}
suite.expectedConfig.Validation.Manifests.Indexes.Platforms = ""
// Note: this also tests that REGISTRY_STORAGE and // Note: this also tests that REGISTRY_STORAGE and
// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together // REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
@ -566,5 +582,11 @@ func copyConfig(config Configuration) *Configuration {
configCopy.Redis = config.Redis configCopy.Redis = config.Redis
configCopy.Validation = Validation{
Enabled: config.Validation.Enabled,
Disabled: config.Validation.Disabled,
Manifests: config.Validation.Manifests,
}
return configCopy return configCopy
} }

View file

@ -288,6 +288,11 @@ validation:
- ^https?://([^/]+\.)*example\.com/ - ^https?://([^/]+\.)*example\.com/
deny: deny:
- ^https?://www\.example\.com/ - ^https?://www\.example\.com/
indexes:
platforms: List
platformlist:
- architecture: amd64
os: linux
``` ```
In some instances a configuration option is **optional** but it contains child In some instances a configuration option is **optional** but it contains child
@ -1160,14 +1165,14 @@ username (such as `batman`) and the password for that username.
```yaml ```yaml
validation: validation:
manifests: disabled: false
urls:
allow:
- ^https?://([^/]+\.)*example\.com/
deny:
- ^https?://www\.example\.com/
``` ```
Use these settings to configure what validation the registry performs on content.
Validation is performed when content is uploaded to the registry. Changing these
settings will not validate content that has already been accepting into the registry.
### `disabled` ### `disabled`
The `disabled` flag disables the other options in the `validation` The `disabled` flag disables the other options in the `validation`
@ -1180,6 +1185,16 @@ Use the `manifests` subsection to configure validation of manifests. If
#### `urls` #### `urls`
```yaml
validation:
manifests:
urls:
allow:
- ^https?://([^/]+\.)*example\.com/
deny:
- ^https?://www\.example\.com/
```
The `allow` and `deny` options are each a list of The `allow` and `deny` options are each a list of
[regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in [regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in
pushed manifests. pushed manifests.
@ -1193,6 +1208,54 @@ one of the `allow` regular expressions **and** one of the following holds:
2. `deny` is set but no URLs within the manifest match any of the `deny` regular 2. `deny` is set but no URLs within the manifest match any of the `deny` regular
expressions. expressions.
#### `indexes`
By default the registry will validate that all platform images exist when an image
index is uploaded to the registry. Disabling this validatation is experimental
because other tooling that uses the registry may expect the image index to be complete.
validation:
manifests:
indexes:
platforms: [all|none|list]
platformlist:
- os: linux
architecture: amd64
Use these settings to configure what validation the registry performs on image
index manifests uploaded to the registry.
##### `platforms`
Set `platformexist` to `all` (the default) to validate all platform images exist.
The registry will validate that the images referenced by the index exist in the
registry before accepting the image index.
Set `platforms` to `none` to disable all validation that images exist when an
image index manifest is uploaded. This allows image lists to be uploaded to the
registry without their associated images. This setting is experimental because
other tooling that uses the registry may expect the image index to be complete.
Set `platforms` to `list` to selectively validate the existence of platforms
within image index manifests. This setting is experimental because other tooling
that uses the registry may expect the image index to be complete.
##### `platformlist`
When `platforms` is set to `list`, set `platformlist` to an array of
platforms to validate. If a platform is included in this the array and in the images
contained within an index, the registry will validate that the platform specific image
exists in the registry before accepting the index. The registry will not validate the
existence of platform specific images in the index that do not appear in the
`platformlist` array.
This parameter does not validate that the configured platforms are included in every
index. If an image index does not include one of the platform specific images configured
in the `platformlist` array, it may still be accepted by the registry.
Each platform is a map with two keys, `os` and `architecture`, as defined in the
[OCI Image Index specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions).
## Example: Development configuration ## Example: Development configuration
You can use this simple example for local development: You can use this simple example for local development:

View file

@ -47,7 +47,7 @@ type ManifestBuilder interface {
AppendReference(dependency Describable) error AppendReference(dependency Describable) error
} }
// ManifestService describes operations on image manifests. // ManifestService describes operations on manifests.
type ManifestService interface { type ManifestService interface {
// Exists returns true if the manifest exists. // Exists returns true if the manifest exists.
Exists(ctx context.Context, dgst digest.Digest) (bool, error) Exists(ctx context.Context, dgst digest.Digest) (bool, error)

View file

@ -2514,7 +2514,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB
func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) { func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
if resp.StatusCode != expectedStatus { if resp.StatusCode != expectedStatus {
t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus) t.Logf("unexpected status %s: expected %v, got %v", msg, resp.StatusCode, expectedStatus)
maybeDumpResponse(t, resp) maybeDumpResponse(t, resp)
t.FailNow() t.FailNow()
} }

View file

@ -255,6 +255,21 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App {
options = append(options, storage.ManifestURLsDenyRegexp(re)) options = append(options, storage.ManifestURLsDenyRegexp(re))
} }
} }
switch config.Validation.Manifests.Indexes.Platforms {
case "list":
options = append(options, storage.EnableValidateImageIndexImagesExist)
for _, platform := range config.Validation.Manifests.Indexes.PlatformList {
options = append(options, storage.AddValidateImageIndexImagesExistPlatform(platform.Architecture, platform.OS))
}
fallthrough
case "none":
dcontext.GetLogger(app).Warn("Image index completeness validation has been disabled, which is an experimental option because other container tooling might expect all image indexes to be complete")
case "all":
fallthrough
default:
options = append(options, storage.EnableValidateImageIndexImagesExist)
}
} }
// configure storage caches // configure storage caches

View file

@ -13,9 +13,10 @@ import (
// manifestListHandler is a ManifestHandler that covers schema2 manifest lists. // manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
type manifestListHandler struct { type manifestListHandler struct {
repository distribution.Repository repository distribution.Repository
blobStore distribution.BlobStore blobStore distribution.BlobStore
ctx context.Context ctx context.Context
validateImageIndexes validateImageIndexes
} }
var _ ManifestHandler = &manifestListHandler{} var _ ManifestHandler = &manifestListHandler{}
@ -74,24 +75,24 @@ func (ms *manifestListHandler) Put(ctx context.Context, manifestList distributio
func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error { func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification var errs distribution.ErrManifestVerification
if !skipDependencyVerification { // Check if we should be validating the existence of any child images in images indexes
// This manifest service is different from the blob service if ms.validateImageIndexes.imagesExist && !skipDependencyVerification {
// returned by Blob. It uses a linked blob store to ensure that // Get the manifest service we can use to check for the existence of child images
// only manifests are accessible.
manifestService, err := ms.repository.Manifests(ctx) manifestService, err := ms.repository.Manifests(ctx)
if err != nil { if err != nil {
return err return err
} }
for _, manifestDescriptor := range mnfst.References() { for _, manifestDescriptor := range mnfst.References() {
exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest) if ms.platformMustExist(manifestDescriptor) {
if err != nil && err != distribution.ErrBlobUnknown { exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
errs = append(errs, err) if err != nil && err != distribution.ErrBlobUnknown {
} errs = append(errs, err)
if err != nil || !exists { }
// On error here, we always append unknown blob errors. if err != nil || !exists {
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest}) // On error here, we always append unknown blob errors.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
}
} }
} }
} }
@ -101,3 +102,24 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
return nil return nil
} }
// platformMustExist checks if a descriptor within an index should be validated as existing before accepting the manifest into the registry.
func (ms *manifestListHandler) platformMustExist(descriptor distribution.Descriptor) bool {
// If there are no image platforms configured to validate, we must check the existence of all child images.
if len(ms.validateImageIndexes.imagePlatforms) == 0 {
return true
}
imagePlatform := descriptor.Platform
// If the platform matches a platform that is configured to validate, we must check the existence.
for _, platform := range ms.validateImageIndexes.imagePlatforms {
if imagePlatform.Architecture == platform.architecture &&
imagePlatform.OS == platform.os {
return true
}
}
// If the platform doesn't match a platform configured to validate, we don't need to check the existence.
return false
}

View file

@ -10,6 +10,7 @@ import (
"github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/manifest" "github.com/distribution/distribution/v3/manifest"
"github.com/distribution/distribution/v3/manifest/manifestlist"
"github.com/distribution/distribution/v3/manifest/ocischema" "github.com/distribution/distribution/v3/manifest/ocischema"
"github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/manifest/schema2"
"github.com/distribution/distribution/v3/registry/storage/cache/memory" "github.com/distribution/distribution/v3/registry/storage/cache/memory"
@ -54,7 +55,7 @@ func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, opt
} }
func TestManifestStorage(t *testing.T) { func TestManifestStorage(t *testing.T) {
testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect) testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
} }
func testManifestStorage(t *testing.T, options ...RegistryOption) { func testManifestStorage(t *testing.T, options ...RegistryOption) {
@ -314,7 +315,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
repoName, _ := reference.WithName("foo/bar") repoName, _ := reference.WithName("foo/bar")
env := newManifestStoreTestEnv(t, repoName, "thetag", env := newManifestStoreTestEnv(t, repoName, "thetag",
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
EnableDelete, EnableRedirect) EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
ctx := context.Background() ctx := context.Background()
ms, err := env.repository.Manifests(ctx) ms, err := env.repository.Manifests(ctx)
@ -322,46 +323,36 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
t.Fatal(err) t.Fatal(err)
} }
// Build a manifest and store it and its layers in the registry // Build a manifest and store its layers in the registry
blobStore := env.repository.Blobs(ctx) blobStore := env.repository.Blobs(ctx)
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{}) mfst, err := createRandomImage(t, testname, imageMediaType, blobStore)
err = builder.(*ocischema.Builder).SetMediaType(imageMediaType)
if err != nil { if err != nil {
t.Fatal(err) t.Fatalf("%s: unexpected error generating random image: %v", testname, err)
} }
// Add some layers // create an image index
for i := 0; i < 2; i++ {
rs, dgst, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("%s: unexpected error generating test layer file", testname)
}
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx) platformSpec := &v1.Platform{
if err != nil { Architecture: "atari2600",
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err) OS: "CP/M",
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
}
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
}
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
t.Fatalf("%s unexpected error appending references: %v", testname, err)
}
} }
mfst, err := builder.Build(ctx) mfstDescriptors := []distribution.Descriptor{
createOciManifestDescriptor(t, testname, mfst, platformSpec),
}
imageIndex, err := ociIndexFromDesriptorsWithMediaType(mfstDescriptors, indexMediaType)
if err != nil { if err != nil {
t.Fatalf("%s: unexpected error generating manifest: %v", testname, err) t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
} }
// before putting the manifest test for proper handling of SchemaVersion _, err = ms.Put(ctx, imageIndex)
if err == nil {
t.Fatalf("%s: expected error putting image index without child manifests in the registry: %v", testname, err)
}
// Test for proper handling of SchemaVersion for the image
if mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion != 2 { if mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion != 2 {
t.Fatalf("%s: unexpected error generating default version for oci manifest", testname) t.Fatalf("%s: unexpected error generating default version for oci manifest", testname)
@ -379,22 +370,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
} }
} }
// Also create an image index that contains the manifest // We can now push the index
descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest)
if err != nil {
t.Fatalf("%s: unexpected error getting manifest descriptor", testname)
}
descriptor.MediaType = v1.MediaTypeImageManifest
descriptor.Platform = &v1.Platform{
Architecture: "atari2600",
OS: "CP/M",
}
imageIndex, err := ociIndexFromDesriptorsWithMediaType([]distribution.Descriptor{descriptor}, indexMediaType)
if err != nil {
t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
}
var indexDigest digest.Digest var indexDigest digest.Digest
if indexDigest, err = ms.Put(ctx, imageIndex); err != nil { if indexDigest, err = ms.Put(ctx, imageIndex); err != nil {
@ -456,6 +432,244 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
} }
} }
func TestIndexManifestStorageWithoutImageCheck(t *testing.T) {
imageMediaType := v1.MediaTypeImageManifest
indexMediaType := v1.MediaTypeImageIndex
repoName, _ := reference.WithName("foo/bar")
env := newManifestStoreTestEnv(t, repoName, "thetag",
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
EnableDelete, EnableRedirect)
ctx := context.Background()
ms, err := env.repository.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
// Build a manifest and store its layers in the registry
blobStore := env.repository.Blobs(ctx)
manifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
if err != nil {
t.Fatalf("unexpected error generating random image: %v", err)
}
// create an image index
ociPlatformSpec := &v1.Platform{
Architecture: "atari2600",
OS: "CP/M",
}
ociManifestDescriptors := []distribution.Descriptor{
createOciManifestDescriptor(t, t.Name(), manifest, ociPlatformSpec),
}
imageIndex, err := ociIndexFromDesriptorsWithMediaType(ociManifestDescriptors, indexMediaType)
if err != nil {
t.Fatalf("unexpected error creating image index: %v", err)
}
// We should be able to put the index without having put the image
_, err = ms.Put(ctx, imageIndex)
if err != nil {
t.Fatalf("unexpected error putting sparse OCI image index: %v", err)
}
// same for a manifest list
listPlatformSpec := &manifestlist.PlatformSpec{
Architecture: "atari2600",
OS: "CP/M",
}
listManifestDescriptors := []manifestlist.ManifestDescriptor{
createManifestListDescriptor(t, t.Name(), manifest, listPlatformSpec),
}
list, err := manifestlist.FromDescriptors(listManifestDescriptors)
if err != nil {
t.Fatalf("unexpected error creating manifest list: %v", err)
}
// We should be able to put the list without having put the image
_, err = ms.Put(ctx, list)
if err != nil {
t.Fatalf("unexpected error putting sparse manifest list: %v", err)
}
}
func TestIndexManifestStorageWithSelectivePlatforms(t *testing.T) {
imageMediaType := v1.MediaTypeImageManifest
indexMediaType := v1.MediaTypeImageIndex
repoName, _ := reference.WithName("foo/bar")
env := newManifestStoreTestEnv(t, repoName, "thetag",
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist,
AddValidateImageIndexImagesExistPlatform("amd64", "linux"))
ctx := context.Background()
ms, err := env.repository.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
// Build a manifests their layers in the registry
blobStore := env.repository.Blobs(ctx)
amdManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
if err != nil {
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
}
armManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
if err != nil {
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
}
atariManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
if err != nil {
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
}
// create an image index
amdPlatformSpec := &v1.Platform{
Architecture: "amd64",
OS: "linux",
}
armPlatformSpec := &v1.Platform{
Architecture: "arm",
OS: "plan9",
}
atariPlatformSpec := &v1.Platform{
Architecture: "atari2600",
OS: "CP/M",
}
manifestDescriptors := []distribution.Descriptor{
createOciManifestDescriptor(t, t.Name(), amdManifest, amdPlatformSpec),
createOciManifestDescriptor(t, t.Name(), armManifest, armPlatformSpec),
createOciManifestDescriptor(t, t.Name(), atariManifest, atariPlatformSpec),
}
imageIndex, err := ociIndexFromDesriptorsWithMediaType(manifestDescriptors, indexMediaType)
if err != nil {
t.Fatalf("unexpected error creating image index: %v", err)
}
// Test we can't push with no image manifests existing in the registry
_, err = ms.Put(ctx, imageIndex)
if err == nil {
t.Fatalf("expected error putting image index without existing images: %v", err)
}
// Test we can't push with a manifest but not the right one
_, err = ms.Put(ctx, atariManifest)
if err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
_, err = ms.Put(ctx, imageIndex)
if err == nil {
t.Fatalf("expected error putting image index without correct existing images: %v", err)
}
// Test we can push with the right manifest
_, err = ms.Put(ctx, amdManifest)
if err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
_, err = ms.Put(ctx, imageIndex)
if err != nil {
t.Fatalf("unexpected error putting image index: %v", err)
}
}
// createRandomImage builds an image manifest and store it and its layers in the registry
func createRandomImage(t *testing.T, testname string, imageMediaType string, blobStore distribution.BlobStore) (distribution.Manifest, error) {
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
err := builder.(*ocischema.Builder).SetMediaType(imageMediaType)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
// Add some layers
for i := 0; i < 2; i++ {
rs, dgst, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("%s: unexpected error generating test layer file", testname)
}
wr, err := blobStore.Create(ctx)
if err != nil {
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
}
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
}
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
t.Fatalf("%s unexpected error appending references: %v", testname, err)
}
}
return builder.Build(ctx)
}
// createOciManifestDescriptor builds a manifest descriptor from a manifest and a platform descriptor
func createOciManifestDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *v1.Platform) distribution.Descriptor {
manifestMediaType, manifestPayload, err := manifest.Payload()
if err != nil {
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
}
manifestDigest := digest.FromBytes(manifestPayload)
return distribution.Descriptor{
Digest: manifestDigest,
Size: int64(len(manifestPayload)),
MediaType: manifestMediaType,
Platform: &v1.Platform{
Architecture: platformSpec.Architecture,
OS: platformSpec.OS,
},
}
}
// createManifestListDescriptor builds a manifest descriptor from a manifest and a platform descriptor
func createManifestListDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *manifestlist.PlatformSpec) manifestlist.ManifestDescriptor {
manifestMediaType, manifestPayload, err := manifest.Payload()
if err != nil {
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
}
manifestDigest := digest.FromBytes(manifestPayload)
return manifestlist.ManifestDescriptor{
Descriptor: distribution.Descriptor{
Digest: manifestDigest,
Size: int64(len(manifestPayload)),
MediaType: manifestMediaType,
},
Platform: manifestlist.PlatformSpec{
Architecture: platformSpec.Architecture,
OS: platformSpec.OS,
},
}
}
// TestLinkPathFuncs ensures that the link path functions behavior are locked // TestLinkPathFuncs ensures that the link path functions behavior are locked
// down and implemented as expected. // down and implemented as expected.
func TestLinkPathFuncs(t *testing.T) { func TestLinkPathFuncs(t *testing.T) {

View file

@ -26,8 +26,11 @@ type registry struct {
tagLookupConcurrencyLimit int tagLookupConcurrencyLimit int
resumableDigestEnabled bool resumableDigestEnabled bool
blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory
manifestURLs manifestURLs
driver storagedriver.StorageDriver driver storagedriver.StorageDriver
// Validation
manifestURLs manifestURLs
validateImageIndexes validateImageIndexes
} }
// manifestURLs holds regular expressions for controlling manifest URL whitelisting // manifestURLs holds regular expressions for controlling manifest URL whitelisting
@ -36,6 +39,20 @@ type manifestURLs struct {
deny *regexp.Regexp deny *regexp.Regexp
} }
// validateImageIndexImages holds configuration for validation of image indexes
type validateImageIndexes struct {
// exist can be used to disable checking that platform images exist entirely. Default true.
imagesExist bool
// platforms can be used to only validate the existence of images for a set of platforms. The empty array means validate all platforms.
imagePlatforms []platform
}
// platform represents a platform to validate exists in the
type platform struct {
architecture string
os string
}
// RegistryOption is the type used for functional options for NewRegistry. // RegistryOption is the type used for functional options for NewRegistry.
type RegistryOption func(*registry) error type RegistryOption func(*registry) error
@ -83,6 +100,28 @@ func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption {
} }
} }
// EnableValidateImageIndexImagesExist is a functional option for NewRegistry. It enables
// validation that references exist before an image index is accepted.
func EnableValidateImageIndexImagesExist(registry *registry) error {
registry.validateImageIndexes.imagesExist = true
return nil
}
// AddValidateImageIndexImagesExistPlatform returns a functional option for NewRegistry.
// It adds a platform to check for existence before an image index is accepted.
func AddValidateImageIndexImagesExistPlatform(architecture string, os string) RegistryOption {
return func(registry *registry) error {
registry.validateImageIndexes.imagePlatforms = append(
registry.validateImageIndexes.imagePlatforms,
platform{
architecture: architecture,
os: os,
},
)
return nil
}
}
// BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the // BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the
// factory to create BlobDescriptorServiceFactory middleware. // factory to create BlobDescriptorServiceFactory middleware.
func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption { func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption {
@ -240,9 +279,10 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
} }
manifestListHandler := &manifestListHandler{ manifestListHandler := &manifestListHandler{
ctx: ctx, ctx: ctx,
repository: repo, repository: repo,
blobStore: blobStore, blobStore: blobStore,
validateImageIndexes: repo.validateImageIndexes,
} }
ms := &manifestStore{ ms := &manifestStore{

View file

@ -18,7 +18,9 @@ func TestVerifyManifestForeignLayer(t *testing.T) {
inmemoryDriver := inmemory.New() inmemoryDriver := inmemory.New()
registry := createRegistry(t, inmemoryDriver, registry := createRegistry(t, inmemoryDriver,
ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")), ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope"))) ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
EnableValidateImageIndexImagesExist,
)
repo := makeRepository(t, registry, "test") repo := makeRepository(t, registry, "test")
manifestService := makeManifestService(t, repo) manifestService := makeManifestService(t, repo)
@ -156,7 +158,9 @@ func TestVerifyManifestBlobLayerAndConfig(t *testing.T) {
inmemoryDriver := inmemory.New() inmemoryDriver := inmemory.New()
registry := createRegistry(t, inmemoryDriver, registry := createRegistry(t, inmemoryDriver,
ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")), ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope"))) ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
EnableValidateImageIndexImagesExist,
)
repo := makeRepository(t, registry, strings.ToLower(t.Name())) repo := makeRepository(t, registry, strings.ToLower(t.Name()))
manifestService := makeManifestService(t, repo) manifestService := makeManifestService(t, repo)