Add option to enable sparse indexes (#3536)
This commit is contained in:
Milos Gajdos
GitHub
commit
37b83869a9
10 changed files with 520 additions and 95 deletions
|
|
@ -181,25 +181,7 @@ type Configuration struct {
|
|||
Proxy Proxy `yaml:"proxy,omitempty"`
|
||||
|
||||
// Validation configures validation options for the registry.
|
||||
Validation struct {
|
||||
// Enabled enables the other options in this section. This field is
|
||||
// deprecated in favor of Disabled.
|
||||
Enabled bool `yaml:"enabled,omitempty"`
|
||||
// Disabled disables the other options in this section.
|
||||
Disabled bool `yaml:"disabled,omitempty"`
|
||||
// Manifests configures manifest validation.
|
||||
Manifests struct {
|
||||
// URLs configures validation for URLs in pushed manifests.
|
||||
URLs struct {
|
||||
// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
|
||||
// that URLs in pushed manifests must match.
|
||||
Allow []string `yaml:"allow,omitempty"`
|
||||
// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
|
||||
// that URLs in pushed manifests must not match.
|
||||
Deny []string `yaml:"deny,omitempty"`
|
||||
} `yaml:"urls,omitempty"`
|
||||
} `yaml:"manifests,omitempty"`
|
||||
} `yaml:"validation,omitempty"`
|
||||
Validation Validation `yaml:"validation,omitempty"`
|
||||
|
||||
// Policy configures registry policy options.
|
||||
Policy struct {
|
||||
|
|
@ -366,6 +348,13 @@ type Health struct {
|
|||
} `yaml:"storagedriver,omitempty"`
|
||||
}
|
||||
|
||||
type Platform struct {
|
||||
// Architecture is the architecture for this platform
|
||||
Architecture string `yaml:"architecture,omitempty"`
|
||||
// OS is the operating system for this platform
|
||||
OS string `yaml:"os,omitempty"`
|
||||
}
|
||||
|
||||
// v0_1Configuration is a Version 0.1 Configuration struct
|
||||
// This is currently aliased to Configuration, as it is the current version
|
||||
type v0_1Configuration Configuration
|
||||
|
|
@ -653,6 +642,62 @@ type Proxy struct {
|
|||
TTL *time.Duration `yaml:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
type Validation struct {
|
||||
// Enabled enables the other options in this section. This field is
|
||||
// deprecated in favor of Disabled.
|
||||
Enabled bool `yaml:"enabled,omitempty"`
|
||||
// Disabled disables the other options in this section.
|
||||
Disabled bool `yaml:"disabled,omitempty"`
|
||||
// Manifests configures manifest validation.
|
||||
Manifests ValidationManifests `yaml:"manifests,omitempty"`
|
||||
}
|
||||
|
||||
type ValidationManifests struct {
|
||||
// URLs configures validation for URLs in pushed manifests.
|
||||
URLs struct {
|
||||
// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
|
||||
// that URLs in pushed manifests must match.
|
||||
Allow []string `yaml:"allow,omitempty"`
|
||||
// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
|
||||
// that URLs in pushed manifests must not match.
|
||||
Deny []string `yaml:"deny,omitempty"`
|
||||
} `yaml:"urls,omitempty"`
|
||||
// ImageIndexes configures validation of image indexes
|
||||
Indexes ValidationIndexes `yaml:"indexes,omitempty"`
|
||||
}
|
||||
|
||||
type ValidationIndexes struct {
|
||||
// Platforms configures the validation applies to the platform images included in an image index
|
||||
Platforms Platforms `yaml:"platforms"`
|
||||
// PlatformList filters the set of platforms to validate for image existence.
|
||||
PlatformList []Platform `yaml:"platformlist,omitempty"`
|
||||
}
|
||||
|
||||
// Platforms configures the validation applies to the platform images included in an image index
|
||||
// This can be all, none, or list
|
||||
type Platforms string
|
||||
|
||||
// UnmarshalYAML implements the yaml.Umarshaler interface
|
||||
// Unmarshals a string into a Platforms option, lowercasing the string and validating that it represents a
|
||||
// valid option
|
||||
func (platforms *Platforms) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var platformsString string
|
||||
err := unmarshal(&platformsString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
platformsString = strings.ToLower(platformsString)
|
||||
switch platformsString {
|
||||
case "all", "none", "list":
|
||||
default:
|
||||
return fmt.Errorf("invalid platforms option %s Must be one of [all, none, list]", platformsString)
|
||||
}
|
||||
|
||||
*platforms = Platforms(platformsString)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse parses an input configuration yaml document into a Configuration struct
|
||||
// This should generally be capable of handling old configuration format versions
|
||||
//
|
||||
|
|
|
|||
|
|
@ -151,6 +151,13 @@ var configStruct = Configuration{
|
|||
ReadTimeout: time.Millisecond * 10,
|
||||
WriteTimeout: time.Millisecond * 10,
|
||||
},
|
||||
Validation: Validation{
|
||||
Manifests: ValidationManifests{
|
||||
Indexes: ValidationIndexes{
|
||||
Platforms: "none",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// configYamlV0_1 is a Version 0.1 yaml document representing configStruct
|
||||
|
|
@ -206,6 +213,10 @@ redis:
|
|||
dialtimeout: 10ms
|
||||
readtimeout: 10ms
|
||||
writetimeout: 10ms
|
||||
validation:
|
||||
manifests:
|
||||
indexes:
|
||||
platforms: none
|
||||
`
|
||||
|
||||
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
|
||||
|
|
@ -235,6 +246,10 @@ notifications:
|
|||
http:
|
||||
headers:
|
||||
X-Content-Type-Options: [nosniff]
|
||||
validation:
|
||||
manifests:
|
||||
indexes:
|
||||
platforms: none
|
||||
`
|
||||
|
||||
type ConfigSuite struct {
|
||||
|
|
@ -295,6 +310,7 @@ func (suite *ConfigSuite) TestParseIncomplete() {
|
|||
suite.expectedConfig.Notifications = Notifications{}
|
||||
suite.expectedConfig.HTTP.Headers = nil
|
||||
suite.expectedConfig.Redis = Redis{}
|
||||
suite.expectedConfig.Validation.Manifests.Indexes.Platforms = ""
|
||||
|
||||
// Note: this also tests that REGISTRY_STORAGE and
|
||||
// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
|
||||
|
|
@ -566,5 +582,11 @@ func copyConfig(config Configuration) *Configuration {
|
|||
|
||||
configCopy.Redis = config.Redis
|
||||
|
||||
configCopy.Validation = Validation{
|
||||
Enabled: config.Validation.Enabled,
|
||||
Disabled: config.Validation.Disabled,
|
||||
Manifests: config.Validation.Manifests,
|
||||
}
|
||||
|
||||
return configCopy
|
||||
}
|
||||
|
|
|
|||
|
|
@ -288,6 +288,11 @@ validation:
|
|||
- ^https?://([^/]+\.)*example\.com/
|
||||
deny:
|
||||
- ^https?://www\.example\.com/
|
||||
indexes:
|
||||
platforms: List
|
||||
platformlist:
|
||||
- architecture: amd64
|
||||
os: linux
|
||||
```
|
||||
|
||||
In some instances a configuration option is **optional** but it contains child
|
||||
|
|
@ -1160,14 +1165,14 @@ username (such as `batman`) and the password for that username.
|
|||
|
||||
```yaml
|
||||
validation:
|
||||
manifests:
|
||||
urls:
|
||||
allow:
|
||||
- ^https?://([^/]+\.)*example\.com/
|
||||
deny:
|
||||
- ^https?://www\.example\.com/
|
||||
disabled: false
|
||||
```
|
||||
|
||||
Use these settings to configure what validation the registry performs on content.
|
||||
|
||||
Validation is performed when content is uploaded to the registry. Changing these
|
||||
settings will not validate content that has already been accepting into the registry.
|
||||
|
||||
### `disabled`
|
||||
|
||||
The `disabled` flag disables the other options in the `validation`
|
||||
|
|
@ -1180,6 +1185,16 @@ Use the `manifests` subsection to configure validation of manifests. If
|
|||
|
||||
#### `urls`
|
||||
|
||||
```yaml
|
||||
validation:
|
||||
manifests:
|
||||
urls:
|
||||
allow:
|
||||
- ^https?://([^/]+\.)*example\.com/
|
||||
deny:
|
||||
- ^https?://www\.example\.com/
|
||||
```
|
||||
|
||||
The `allow` and `deny` options are each a list of
|
||||
[regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in
|
||||
pushed manifests.
|
||||
|
|
@ -1193,6 +1208,54 @@ one of the `allow` regular expressions **and** one of the following holds:
|
|||
2. `deny` is set but no URLs within the manifest match any of the `deny` regular
|
||||
expressions.
|
||||
|
||||
#### `indexes`
|
||||
|
||||
By default the registry will validate that all platform images exist when an image
|
||||
index is uploaded to the registry. Disabling this validatation is experimental
|
||||
because other tooling that uses the registry may expect the image index to be complete.
|
||||
|
||||
validation:
|
||||
manifests:
|
||||
indexes:
|
||||
platforms: [all|none|list]
|
||||
platformlist:
|
||||
- os: linux
|
||||
architecture: amd64
|
||||
|
||||
Use these settings to configure what validation the registry performs on image
|
||||
index manifests uploaded to the registry.
|
||||
|
||||
##### `platforms`
|
||||
|
||||
Set `platformexist` to `all` (the default) to validate all platform images exist.
|
||||
The registry will validate that the images referenced by the index exist in the
|
||||
registry before accepting the image index.
|
||||
|
||||
Set `platforms` to `none` to disable all validation that images exist when an
|
||||
image index manifest is uploaded. This allows image lists to be uploaded to the
|
||||
registry without their associated images. This setting is experimental because
|
||||
other tooling that uses the registry may expect the image index to be complete.
|
||||
|
||||
Set `platforms` to `list` to selectively validate the existence of platforms
|
||||
within image index manifests. This setting is experimental because other tooling
|
||||
that uses the registry may expect the image index to be complete.
|
||||
|
||||
##### `platformlist`
|
||||
|
||||
When `platforms` is set to `list`, set `platformlist` to an array of
|
||||
platforms to validate. If a platform is included in this the array and in the images
|
||||
contained within an index, the registry will validate that the platform specific image
|
||||
exists in the registry before accepting the index. The registry will not validate the
|
||||
existence of platform specific images in the index that do not appear in the
|
||||
`platformlist` array.
|
||||
|
||||
This parameter does not validate that the configured platforms are included in every
|
||||
index. If an image index does not include one of the platform specific images configured
|
||||
in the `platformlist` array, it may still be accepted by the registry.
|
||||
|
||||
Each platform is a map with two keys, `os` and `architecture`, as defined in the
|
||||
[OCI Image Index specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions).
|
||||
|
||||
## Example: Development configuration
|
||||
|
||||
You can use this simple example for local development:
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ type ManifestBuilder interface {
|
|||
AppendReference(dependency Describable) error
|
||||
}
|
||||
|
||||
// ManifestService describes operations on image manifests.
|
||||
// ManifestService describes operations on manifests.
|
||||
type ManifestService interface {
|
||||
// Exists returns true if the manifest exists.
|
||||
Exists(ctx context.Context, dgst digest.Digest) (bool, error)
|
||||
|
|
|
|||
|
|
@ -2514,7 +2514,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB
|
|||
|
||||
func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
|
||||
if resp.StatusCode != expectedStatus {
|
||||
t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus)
|
||||
t.Logf("unexpected status %s: expected %v, got %v", msg, resp.StatusCode, expectedStatus)
|
||||
maybeDumpResponse(t, resp)
|
||||
t.FailNow()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -255,6 +255,21 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App {
|
|||
options = append(options, storage.ManifestURLsDenyRegexp(re))
|
||||
}
|
||||
}
|
||||
|
||||
switch config.Validation.Manifests.Indexes.Platforms {
|
||||
case "list":
|
||||
options = append(options, storage.EnableValidateImageIndexImagesExist)
|
||||
for _, platform := range config.Validation.Manifests.Indexes.PlatformList {
|
||||
options = append(options, storage.AddValidateImageIndexImagesExistPlatform(platform.Architecture, platform.OS))
|
||||
}
|
||||
fallthrough
|
||||
case "none":
|
||||
dcontext.GetLogger(app).Warn("Image index completeness validation has been disabled, which is an experimental option because other container tooling might expect all image indexes to be complete")
|
||||
case "all":
|
||||
fallthrough
|
||||
default:
|
||||
options = append(options, storage.EnableValidateImageIndexImagesExist)
|
||||
}
|
||||
}
|
||||
|
||||
// configure storage caches
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ type manifestListHandler struct {
|
|||
repository distribution.Repository
|
||||
blobStore distribution.BlobStore
|
||||
ctx context.Context
|
||||
validateImageIndexes validateImageIndexes
|
||||
}
|
||||
|
||||
var _ ManifestHandler = &manifestListHandler{}
|
||||
|
|
@ -74,17 +75,16 @@ func (ms *manifestListHandler) Put(ctx context.Context, manifestList distributio
|
|||
func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error {
|
||||
var errs distribution.ErrManifestVerification
|
||||
|
||||
if !skipDependencyVerification {
|
||||
// This manifest service is different from the blob service
|
||||
// returned by Blob. It uses a linked blob store to ensure that
|
||||
// only manifests are accessible.
|
||||
|
||||
// Check if we should be validating the existence of any child images in images indexes
|
||||
if ms.validateImageIndexes.imagesExist && !skipDependencyVerification {
|
||||
// Get the manifest service we can use to check for the existence of child images
|
||||
manifestService, err := ms.repository.Manifests(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, manifestDescriptor := range mnfst.References() {
|
||||
if ms.platformMustExist(manifestDescriptor) {
|
||||
exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
|
||||
if err != nil && err != distribution.ErrBlobUnknown {
|
||||
errs = append(errs, err)
|
||||
|
|
@ -95,9 +95,31 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// platformMustExist checks if a descriptor within an index should be validated as existing before accepting the manifest into the registry.
|
||||
func (ms *manifestListHandler) platformMustExist(descriptor distribution.Descriptor) bool {
|
||||
// If there are no image platforms configured to validate, we must check the existence of all child images.
|
||||
if len(ms.validateImageIndexes.imagePlatforms) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
imagePlatform := descriptor.Platform
|
||||
|
||||
// If the platform matches a platform that is configured to validate, we must check the existence.
|
||||
for _, platform := range ms.validateImageIndexes.imagePlatforms {
|
||||
if imagePlatform.Architecture == platform.architecture &&
|
||||
imagePlatform.OS == platform.os {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// If the platform doesn't match a platform configured to validate, we don't need to check the existence.
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/distribution/distribution/v3"
|
||||
"github.com/distribution/distribution/v3/manifest"
|
||||
"github.com/distribution/distribution/v3/manifest/manifestlist"
|
||||
"github.com/distribution/distribution/v3/manifest/ocischema"
|
||||
"github.com/distribution/distribution/v3/manifest/schema2"
|
||||
"github.com/distribution/distribution/v3/registry/storage/cache/memory"
|
||||
|
|
@ -54,7 +55,7 @@ func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, opt
|
|||
}
|
||||
|
||||
func TestManifestStorage(t *testing.T) {
|
||||
testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect)
|
||||
testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
|
||||
}
|
||||
|
||||
func testManifestStorage(t *testing.T, options ...RegistryOption) {
|
||||
|
|
@ -314,7 +315,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
|
|||
repoName, _ := reference.WithName("foo/bar")
|
||||
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
||||
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
||||
EnableDelete, EnableRedirect)
|
||||
EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
|
||||
|
||||
ctx := context.Background()
|
||||
ms, err := env.repository.Manifests(ctx)
|
||||
|
|
@ -322,46 +323,36 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Build a manifest and store it and its layers in the registry
|
||||
// Build a manifest and store its layers in the registry
|
||||
|
||||
blobStore := env.repository.Blobs(ctx)
|
||||
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
|
||||
err = builder.(*ocischema.Builder).SetMediaType(imageMediaType)
|
||||
mfst, err := createRandomImage(t, testname, imageMediaType, blobStore)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("%s: unexpected error generating random image: %v", testname, err)
|
||||
}
|
||||
|
||||
// Add some layers
|
||||
for i := 0; i < 2; i++ {
|
||||
rs, dgst, err := testutil.CreateRandomTarFile()
|
||||
// create an image index
|
||||
|
||||
platformSpec := &v1.Platform{
|
||||
Architecture: "atari2600",
|
||||
OS: "CP/M",
|
||||
}
|
||||
|
||||
mfstDescriptors := []distribution.Descriptor{
|
||||
createOciManifestDescriptor(t, testname, mfst, platformSpec),
|
||||
}
|
||||
|
||||
imageIndex, err := ociIndexFromDesriptorsWithMediaType(mfstDescriptors, indexMediaType)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating test layer file", testname)
|
||||
t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
|
||||
}
|
||||
|
||||
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
|
||||
_, err = ms.Put(ctx, imageIndex)
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected error putting image index without child manifests in the registry: %v", testname, err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(wr, rs); err != nil {
|
||||
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
|
||||
}
|
||||
|
||||
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
|
||||
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
|
||||
}
|
||||
|
||||
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
|
||||
t.Fatalf("%s unexpected error appending references: %v", testname, err)
|
||||
}
|
||||
}
|
||||
|
||||
mfst, err := builder.Build(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating manifest: %v", testname, err)
|
||||
}
|
||||
|
||||
// before putting the manifest test for proper handling of SchemaVersion
|
||||
// Test for proper handling of SchemaVersion for the image
|
||||
|
||||
if mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion != 2 {
|
||||
t.Fatalf("%s: unexpected error generating default version for oci manifest", testname)
|
||||
|
|
@ -379,22 +370,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
|
|||
}
|
||||
}
|
||||
|
||||
// Also create an image index that contains the manifest
|
||||
|
||||
descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error getting manifest descriptor", testname)
|
||||
}
|
||||
descriptor.MediaType = v1.MediaTypeImageManifest
|
||||
descriptor.Platform = &v1.Platform{
|
||||
Architecture: "atari2600",
|
||||
OS: "CP/M",
|
||||
}
|
||||
|
||||
imageIndex, err := ociIndexFromDesriptorsWithMediaType([]distribution.Descriptor{descriptor}, indexMediaType)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
|
||||
}
|
||||
// We can now push the index
|
||||
|
||||
var indexDigest digest.Digest
|
||||
if indexDigest, err = ms.Put(ctx, imageIndex); err != nil {
|
||||
|
|
@ -456,6 +432,244 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
|
|||
}
|
||||
}
|
||||
|
||||
func TestIndexManifestStorageWithoutImageCheck(t *testing.T) {
|
||||
imageMediaType := v1.MediaTypeImageManifest
|
||||
indexMediaType := v1.MediaTypeImageIndex
|
||||
|
||||
repoName, _ := reference.WithName("foo/bar")
|
||||
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
||||
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
||||
EnableDelete, EnableRedirect)
|
||||
|
||||
ctx := context.Background()
|
||||
ms, err := env.repository.Manifests(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Build a manifest and store its layers in the registry
|
||||
|
||||
blobStore := env.repository.Blobs(ctx)
|
||||
manifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error generating random image: %v", err)
|
||||
}
|
||||
|
||||
// create an image index
|
||||
|
||||
ociPlatformSpec := &v1.Platform{
|
||||
Architecture: "atari2600",
|
||||
OS: "CP/M",
|
||||
}
|
||||
|
||||
ociManifestDescriptors := []distribution.Descriptor{
|
||||
createOciManifestDescriptor(t, t.Name(), manifest, ociPlatformSpec),
|
||||
}
|
||||
|
||||
imageIndex, err := ociIndexFromDesriptorsWithMediaType(ociManifestDescriptors, indexMediaType)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating image index: %v", err)
|
||||
}
|
||||
|
||||
// We should be able to put the index without having put the image
|
||||
|
||||
_, err = ms.Put(ctx, imageIndex)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error putting sparse OCI image index: %v", err)
|
||||
}
|
||||
|
||||
// same for a manifest list
|
||||
|
||||
listPlatformSpec := &manifestlist.PlatformSpec{
|
||||
Architecture: "atari2600",
|
||||
OS: "CP/M",
|
||||
}
|
||||
|
||||
listManifestDescriptors := []manifestlist.ManifestDescriptor{
|
||||
createManifestListDescriptor(t, t.Name(), manifest, listPlatformSpec),
|
||||
}
|
||||
|
||||
list, err := manifestlist.FromDescriptors(listManifestDescriptors)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating manifest list: %v", err)
|
||||
}
|
||||
|
||||
// We should be able to put the list without having put the image
|
||||
|
||||
_, err = ms.Put(ctx, list)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error putting sparse manifest list: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexManifestStorageWithSelectivePlatforms(t *testing.T) {
|
||||
imageMediaType := v1.MediaTypeImageManifest
|
||||
indexMediaType := v1.MediaTypeImageIndex
|
||||
|
||||
repoName, _ := reference.WithName("foo/bar")
|
||||
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
||||
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
||||
EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist,
|
||||
AddValidateImageIndexImagesExistPlatform("amd64", "linux"))
|
||||
|
||||
ctx := context.Background()
|
||||
ms, err := env.repository.Manifests(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Build a manifests their layers in the registry
|
||||
|
||||
blobStore := env.repository.Blobs(ctx)
|
||||
amdManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
||||
}
|
||||
armManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
||||
}
|
||||
atariManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
||||
}
|
||||
|
||||
// create an image index
|
||||
|
||||
amdPlatformSpec := &v1.Platform{
|
||||
Architecture: "amd64",
|
||||
OS: "linux",
|
||||
}
|
||||
armPlatformSpec := &v1.Platform{
|
||||
Architecture: "arm",
|
||||
OS: "plan9",
|
||||
}
|
||||
atariPlatformSpec := &v1.Platform{
|
||||
Architecture: "atari2600",
|
||||
OS: "CP/M",
|
||||
}
|
||||
|
||||
manifestDescriptors := []distribution.Descriptor{
|
||||
createOciManifestDescriptor(t, t.Name(), amdManifest, amdPlatformSpec),
|
||||
createOciManifestDescriptor(t, t.Name(), armManifest, armPlatformSpec),
|
||||
createOciManifestDescriptor(t, t.Name(), atariManifest, atariPlatformSpec),
|
||||
}
|
||||
|
||||
imageIndex, err := ociIndexFromDesriptorsWithMediaType(manifestDescriptors, indexMediaType)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating image index: %v", err)
|
||||
}
|
||||
|
||||
// Test we can't push with no image manifests existing in the registry
|
||||
|
||||
_, err = ms.Put(ctx, imageIndex)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error putting image index without existing images: %v", err)
|
||||
}
|
||||
|
||||
// Test we can't push with a manifest but not the right one
|
||||
|
||||
_, err = ms.Put(ctx, atariManifest)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error putting manifest: %v", err)
|
||||
}
|
||||
|
||||
_, err = ms.Put(ctx, imageIndex)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error putting image index without correct existing images: %v", err)
|
||||
}
|
||||
|
||||
// Test we can push with the right manifest
|
||||
|
||||
_, err = ms.Put(ctx, amdManifest)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error putting manifest: %v", err)
|
||||
}
|
||||
|
||||
_, err = ms.Put(ctx, imageIndex)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error putting image index: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// createRandomImage builds an image manifest and store it and its layers in the registry
|
||||
func createRandomImage(t *testing.T, testname string, imageMediaType string, blobStore distribution.BlobStore) (distribution.Manifest, error) {
|
||||
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
|
||||
err := builder.(*ocischema.Builder).SetMediaType(imageMediaType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Add some layers
|
||||
for i := 0; i < 2; i++ {
|
||||
rs, dgst, err := testutil.CreateRandomTarFile()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error generating test layer file", testname)
|
||||
}
|
||||
|
||||
wr, err := blobStore.Create(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(wr, rs); err != nil {
|
||||
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
|
||||
}
|
||||
|
||||
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
|
||||
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
|
||||
}
|
||||
|
||||
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
|
||||
t.Fatalf("%s unexpected error appending references: %v", testname, err)
|
||||
}
|
||||
}
|
||||
|
||||
return builder.Build(ctx)
|
||||
}
|
||||
|
||||
// createOciManifestDescriptor builds a manifest descriptor from a manifest and a platform descriptor
|
||||
func createOciManifestDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *v1.Platform) distribution.Descriptor {
|
||||
manifestMediaType, manifestPayload, err := manifest.Payload()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
|
||||
}
|
||||
manifestDigest := digest.FromBytes(manifestPayload)
|
||||
|
||||
return distribution.Descriptor{
|
||||
Digest: manifestDigest,
|
||||
Size: int64(len(manifestPayload)),
|
||||
MediaType: manifestMediaType,
|
||||
Platform: &v1.Platform{
|
||||
Architecture: platformSpec.Architecture,
|
||||
OS: platformSpec.OS,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// createManifestListDescriptor builds a manifest descriptor from a manifest and a platform descriptor
|
||||
func createManifestListDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *manifestlist.PlatformSpec) manifestlist.ManifestDescriptor {
|
||||
manifestMediaType, manifestPayload, err := manifest.Payload()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
|
||||
}
|
||||
manifestDigest := digest.FromBytes(manifestPayload)
|
||||
|
||||
return manifestlist.ManifestDescriptor{
|
||||
Descriptor: distribution.Descriptor{
|
||||
Digest: manifestDigest,
|
||||
Size: int64(len(manifestPayload)),
|
||||
MediaType: manifestMediaType,
|
||||
},
|
||||
Platform: manifestlist.PlatformSpec{
|
||||
Architecture: platformSpec.Architecture,
|
||||
OS: platformSpec.OS,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestLinkPathFuncs ensures that the link path functions behavior are locked
|
||||
// down and implemented as expected.
|
||||
func TestLinkPathFuncs(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -26,8 +26,11 @@ type registry struct {
|
|||
tagLookupConcurrencyLimit int
|
||||
resumableDigestEnabled bool
|
||||
blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory
|
||||
manifestURLs manifestURLs
|
||||
driver storagedriver.StorageDriver
|
||||
|
||||
// Validation
|
||||
manifestURLs manifestURLs
|
||||
validateImageIndexes validateImageIndexes
|
||||
}
|
||||
|
||||
// manifestURLs holds regular expressions for controlling manifest URL whitelisting
|
||||
|
|
@ -36,6 +39,20 @@ type manifestURLs struct {
|
|||
deny *regexp.Regexp
|
||||
}
|
||||
|
||||
// validateImageIndexImages holds configuration for validation of image indexes
|
||||
type validateImageIndexes struct {
|
||||
// exist can be used to disable checking that platform images exist entirely. Default true.
|
||||
imagesExist bool
|
||||
// platforms can be used to only validate the existence of images for a set of platforms. The empty array means validate all platforms.
|
||||
imagePlatforms []platform
|
||||
}
|
||||
|
||||
// platform represents a platform to validate exists in the
|
||||
type platform struct {
|
||||
architecture string
|
||||
os string
|
||||
}
|
||||
|
||||
// RegistryOption is the type used for functional options for NewRegistry.
|
||||
type RegistryOption func(*registry) error
|
||||
|
||||
|
|
@ -83,6 +100,28 @@ func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption {
|
|||
}
|
||||
}
|
||||
|
||||
// EnableValidateImageIndexImagesExist is a functional option for NewRegistry. It enables
|
||||
// validation that references exist before an image index is accepted.
|
||||
func EnableValidateImageIndexImagesExist(registry *registry) error {
|
||||
registry.validateImageIndexes.imagesExist = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddValidateImageIndexImagesExistPlatform returns a functional option for NewRegistry.
|
||||
// It adds a platform to check for existence before an image index is accepted.
|
||||
func AddValidateImageIndexImagesExistPlatform(architecture string, os string) RegistryOption {
|
||||
return func(registry *registry) error {
|
||||
registry.validateImageIndexes.imagePlatforms = append(
|
||||
registry.validateImageIndexes.imagePlatforms,
|
||||
platform{
|
||||
architecture: architecture,
|
||||
os: os,
|
||||
},
|
||||
)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the
|
||||
// factory to create BlobDescriptorServiceFactory middleware.
|
||||
func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption {
|
||||
|
|
@ -243,6 +282,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
|
|||
ctx: ctx,
|
||||
repository: repo,
|
||||
blobStore: blobStore,
|
||||
validateImageIndexes: repo.validateImageIndexes,
|
||||
}
|
||||
|
||||
ms := &manifestStore{
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ func TestVerifyManifestForeignLayer(t *testing.T) {
|
|||
inmemoryDriver := inmemory.New()
|
||||
registry := createRegistry(t, inmemoryDriver,
|
||||
ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
|
||||
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")))
|
||||
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
|
||||
EnableValidateImageIndexImagesExist,
|
||||
)
|
||||
repo := makeRepository(t, registry, "test")
|
||||
manifestService := makeManifestService(t, repo)
|
||||
|
||||
|
|
@ -156,7 +158,9 @@ func TestVerifyManifestBlobLayerAndConfig(t *testing.T) {
|
|||
inmemoryDriver := inmemory.New()
|
||||
registry := createRegistry(t, inmemoryDriver,
|
||||
ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
|
||||
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")))
|
||||
ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
|
||||
EnableValidateImageIndexImagesExist,
|
||||
)
|
||||
|
||||
repo := makeRepository(t, registry, strings.ToLower(t.Name()))
|
||||
manifestService := makeManifestService(t, repo)
|
||||
|
|
|
|||
Loading…
Reference in a new issue