Add option to enable sparse indexes (#3536)

configuration/configuration.go

@@ -181,25 +181,7 @@ type Configuration struct {
 	Proxy Proxy `yaml:"proxy,omitempty"`
 
 	// Validation configures validation options for the registry.
-	Validation struct {
+	Validation Validation `yaml:"validation,omitempty"`
-		// Enabled enables the other options in this section. This field is
-		// deprecated in favor of Disabled.
-		Enabled bool `yaml:"enabled,omitempty"`
-		// Disabled disables the other options in this section.
-		Disabled bool `yaml:"disabled,omitempty"`
-		// Manifests configures manifest validation.
-		Manifests struct {
-			// URLs configures validation for URLs in pushed manifests.
-			URLs struct {
-				// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must match.
-				Allow []string `yaml:"allow,omitempty"`
-				// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must not match.
-				Deny []string `yaml:"deny,omitempty"`
-			} `yaml:"urls,omitempty"`
-		} `yaml:"manifests,omitempty"`
-	} `yaml:"validation,omitempty"`
 
 	// Policy configures registry policy options.
 	Policy struct {
@@ -366,6 +348,13 @@ type Health struct {
 	} `yaml:"storagedriver,omitempty"`
 }
 
+type Platform struct {
+	// Architecture is the architecture for this platform
+	Architecture string `yaml:"architecture,omitempty"`
+	// OS is the operating system for this platform
+	OS string `yaml:"os,omitempty"`
+}
+
 // v0_1Configuration is a Version 0.1 Configuration struct
 // This is currently aliased to Configuration, as it is the current version
 type v0_1Configuration Configuration
@@ -653,6 +642,62 @@ type Proxy struct {
 	TTL *time.Duration `yaml:"ttl,omitempty"`
 }
 
+type Validation struct {
+	// Enabled enables the other options in this section. This field is
+	// deprecated in favor of Disabled.
+	Enabled bool `yaml:"enabled,omitempty"`
+	// Disabled disables the other options in this section.
+	Disabled bool `yaml:"disabled,omitempty"`
+	// Manifests configures manifest validation.
+	Manifests ValidationManifests `yaml:"manifests,omitempty"`
+}
+
+type ValidationManifests struct {
+	// URLs configures validation for URLs in pushed manifests.
+	URLs struct {
+		// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must match.
+		Allow []string `yaml:"allow,omitempty"`
+		// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must not match.
+		Deny []string `yaml:"deny,omitempty"`
+	} `yaml:"urls,omitempty"`
+	// ImageIndexes configures validation of image indexes
+	Indexes ValidationIndexes `yaml:"indexes,omitempty"`
+}
+
+type ValidationIndexes struct {
+	// Platforms configures the validation applies to the platform images included in an image index
+	Platforms Platforms `yaml:"platforms"`
+	// PlatformList filters the set of platforms to validate for image existence.
+	PlatformList []Platform `yaml:"platformlist,omitempty"`
+}
+
+// Platforms configures the validation applies to the platform images included in an image index
+// This can be all, none, or list
+type Platforms string
+
+// UnmarshalYAML implements the yaml.Umarshaler interface
+// Unmarshals a string into a Platforms option, lowercasing the string and validating that it represents a
+// valid option
+func (platforms *Platforms) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var platformsString string
+	err := unmarshal(&platformsString)
+	if err != nil {
+		return err
+	}
+
+	platformsString = strings.ToLower(platformsString)
+	switch platformsString {
+	case "all", "none", "list":
+	default:
+		return fmt.Errorf("invalid platforms option %s Must be one of [all, none, list]", platformsString)
+	}
+
+	*platforms = Platforms(platformsString)
+	return nil
+}
+
 // Parse parses an input configuration yaml document into a Configuration struct
 // This should generally be capable of handling old configuration format versions
 //

configuration/configuration_test.go

@@ -151,6 +151,13 @@ var configStruct = Configuration{
 		ReadTimeout:  time.Millisecond * 10,
 		WriteTimeout: time.Millisecond * 10,
 	},
+	Validation: Validation{
+		Manifests: ValidationManifests{
+			Indexes: ValidationIndexes{
+				Platforms: "none",
+			},
+		},
+	},
 }
 
 // configYamlV0_1 is a Version 0.1 yaml document representing configStruct
@@ -206,6 +213,10 @@ redis:
   dialtimeout: 10ms
   readtimeout: 10ms
   writetimeout: 10ms
+validation:
+  manifests:
+    indexes:
+      platforms: none
 `
 
 // inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
@@ -235,6 +246,10 @@ notifications:
 http:
   headers:
     X-Content-Type-Options: [nosniff]
+validation:
+  manifests:
+    indexes:
+      platforms: none
 `
 
 type ConfigSuite struct {
@@ -295,6 +310,7 @@ func (suite *ConfigSuite) TestParseIncomplete() {
 	suite.expectedConfig.Notifications = Notifications{}
 	suite.expectedConfig.HTTP.Headers = nil
 	suite.expectedConfig.Redis = Redis{}
+	suite.expectedConfig.Validation.Manifests.Indexes.Platforms = ""
 
 	// Note: this also tests that REGISTRY_STORAGE and
 	// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
@@ -566,5 +582,11 @@ func copyConfig(config Configuration) *Configuration {
 
 	configCopy.Redis = config.Redis
 
+	configCopy.Validation = Validation{
+		Enabled:   config.Validation.Enabled,
+		Disabled:  config.Validation.Disabled,
+		Manifests: config.Validation.Manifests,
+	}
+
 	return configCopy
 }

docs/content/about/configuration.md

@@ -288,6 +288,11 @@ validation:
         - ^https?://([^/]+\.)*example\.com/
       deny:
         - ^https?://www\.example\.com/
+    indexes:
+      platforms: List
+      platformlist:
+      - architecture: amd64
+        os: linux
 ```
 
 In some instances a configuration option is **optional** but it contains child
@@ -1160,14 +1165,14 @@ username (such as `batman`) and the password for that username.
 
 ```yaml
 validation:
-  manifests:
+  disabled: false
-    urls:
-      allow:
-        - ^https?://([^/]+\.)*example\.com/
-      deny:
-        - ^https?://www\.example\.com/
 ```
 
+Use these settings to configure what validation the registry performs on content.
+
+Validation is performed when content is uploaded to the registry. Changing these
+settings will not validate content that has already been accepting into the registry.
+
 ### `disabled`
 
 The `disabled` flag disables the other options in the `validation`
@@ -1180,6 +1185,16 @@ Use the `manifests` subsection to configure validation of manifests. If
 
 #### `urls`
 
+```yaml
+validation:
+  manifests:
+    urls:
+      allow:
+        - ^https?://([^/]+\.)*example\.com/
+      deny:
+        - ^https?://www\.example\.com/
+```
+
 The `allow` and `deny` options are each a list of
 [regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in
 pushed manifests.
@@ -1193,6 +1208,54 @@ one of the `allow` regular expressions **and** one of the following holds:
 2. `deny` is set but no URLs within the manifest match any of the `deny` regular
    expressions.
 
+#### `indexes`
+
+By default the registry will validate that all platform images exist when an image
+index is uploaded to the registry. Disabling this validatation is experimental
+because other tooling that uses the registry may expect the image index to be complete.
+
+validation:
+  manifests:
+    indexes:
+      platforms: [all|none|list]
+      platformlist:
+      - os: linux
+        architecture: amd64
+
+Use these settings to configure what validation the registry performs on image
+index manifests uploaded to the registry.
+
+##### `platforms`
+
+Set `platformexist` to `all` (the default) to validate all platform images exist.
+The registry will validate that the images referenced by the index exist in the
+registry before accepting the image index.
+
+Set `platforms` to `none` to disable all validation that images exist when an
+image index manifest is uploaded. This allows image lists to be uploaded to the
+registry without their associated images. This setting is experimental because
+other tooling that uses the registry may expect the image index to be complete.
+
+Set `platforms` to `list` to selectively validate the existence of platforms
+within image index manifests. This setting is experimental because other tooling
+that uses the registry may expect the image index to be complete.
+
+##### `platformlist`
+
+When `platforms` is set to `list`, set `platformlist` to an array of
+platforms to validate. If a platform is included in this the array and in the images
+contained within an index, the registry will validate that the platform specific image
+exists in the registry before accepting the index. The registry will not validate the
+existence of platform specific images in the index that do not appear in the
+`platformlist` array.
+
+This parameter does not validate that the configured platforms are included in every
+index. If an image index does not include one of the platform specific images configured
+in the `platformlist` array, it may still be accepted by the registry.
+
+Each platform is a map with two keys, `os` and `architecture`, as defined in the
+[OCI Image Index specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions).
+
 ## Example: Development configuration
 
 You can use this simple example for local development:

manifests.go

@@ -47,7 +47,7 @@ type ManifestBuilder interface {
 	AppendReference(dependency Describable) error
 }
 
-// ManifestService describes operations on image manifests.
+// ManifestService describes operations on manifests.
 type ManifestService interface {
 	// Exists returns true if the manifest exists.
 	Exists(ctx context.Context, dgst digest.Digest) (bool, error)

registry/handlers/api_test.go

@@ -2514,7 +2514,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB
 
 func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
 	if resp.StatusCode != expectedStatus {
-		t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus)
+		t.Logf("unexpected status %s: expected %v, got %v", msg, resp.StatusCode, expectedStatus)
 		maybeDumpResponse(t, resp)
 		t.FailNow()
 	}

registry/handlers/app.go

@@ -255,6 +255,21 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App {
 				options = append(options, storage.ManifestURLsDenyRegexp(re))
 			}
 		}
+
+		switch config.Validation.Manifests.Indexes.Platforms {
+		case "list":
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+			for _, platform := range config.Validation.Manifests.Indexes.PlatformList {
+				options = append(options, storage.AddValidateImageIndexImagesExistPlatform(platform.Architecture, platform.OS))
+			}
+			fallthrough
+		case "none":
+			dcontext.GetLogger(app).Warn("Image index completeness validation has been disabled, which is an experimental option because other container tooling might expect all image indexes to be complete")
+		case "all":
+			fallthrough
+		default:
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+		}
 	}
 
 	// configure storage caches

registry/storage/manifestlisthandler.go

@@ -16,6 +16,7 @@ type manifestListHandler struct {
 	repository           distribution.Repository
 	blobStore            distribution.BlobStore
 	ctx                  context.Context
+	validateImageIndexes validateImageIndexes
 }
 
 var _ ManifestHandler = &manifestListHandler{}
@@ -74,17 +75,16 @@ func (ms *manifestListHandler) Put(ctx context.Context, manifestList distributio
 func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error {
 	var errs distribution.ErrManifestVerification
 
-	if !skipDependencyVerification {
+	// Check if we should be validating the existence of any child images in images indexes
-		// This manifest service is different from the blob service
+	if ms.validateImageIndexes.imagesExist && !skipDependencyVerification {
-		// returned by Blob. It uses a linked blob store to ensure that
+		// Get the manifest service we can use to check for the existence of child images
-		// only manifests are accessible.
-
 		manifestService, err := ms.repository.Manifests(ctx)
 		if err != nil {
 			return err
 		}
 
 		for _, manifestDescriptor := range mnfst.References() {
+			if ms.platformMustExist(manifestDescriptor) {
 				exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
 				if err != nil && err != distribution.ErrBlobUnknown {
 					errs = append(errs, err)
@@ -95,9 +95,31 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
 				}
 			}
 		}
+	}
 	if len(errs) != 0 {
 		return errs
 	}
 
 	return nil
 }
+
+// platformMustExist checks if a descriptor within an index should be validated as existing before accepting the manifest into the registry.
+func (ms *manifestListHandler) platformMustExist(descriptor distribution.Descriptor) bool {
+	// If there are no image platforms configured to validate, we must check the existence of all child images.
+	if len(ms.validateImageIndexes.imagePlatforms) == 0 {
+		return true
+	}
+
+	imagePlatform := descriptor.Platform
+
+	// If the platform matches a platform that is configured to validate, we must check the existence.
+	for _, platform := range ms.validateImageIndexes.imagePlatforms {
+		if imagePlatform.Architecture == platform.architecture &&
+			imagePlatform.OS == platform.os {
+			return true
+		}
+	}
+
+	// If the platform doesn't match a platform configured to validate, we don't need to check the existence.
+	return false
+}

registry/storage/manifeststore_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/distribution/distribution/v3"
 	"github.com/distribution/distribution/v3/manifest"
+	"github.com/distribution/distribution/v3/manifest/manifestlist"
 	"github.com/distribution/distribution/v3/manifest/ocischema"
 	"github.com/distribution/distribution/v3/manifest/schema2"
 	"github.com/distribution/distribution/v3/registry/storage/cache/memory"
@@ -54,7 +55,7 @@ func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, opt
 }
 
 func TestManifestStorage(t *testing.T) {
-	testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect)
+	testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
 }
 
 func testManifestStorage(t *testing.T, options ...RegistryOption) {
@@ -314,7 +315,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
 	repoName, _ := reference.WithName("foo/bar")
 	env := newManifestStoreTestEnv(t, repoName, "thetag",
 		BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
-		EnableDelete, EnableRedirect)
+		EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
 
 	ctx := context.Background()
 	ms, err := env.repository.Manifests(ctx)
@@ -322,46 +323,36 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
 		t.Fatal(err)
 	}
 
-	// Build a manifest and store it and its layers in the registry
+	// Build a manifest and store its layers in the registry
 
 	blobStore := env.repository.Blobs(ctx)
-	builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
+	mfst, err := createRandomImage(t, testname, imageMediaType, blobStore)
-	err = builder.(*ocischema.Builder).SetMediaType(imageMediaType)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("%s: unexpected error generating random image: %v", testname, err)
 	}
 
-	// Add some layers
+	// create an image index
-	for i := 0; i < 2; i++ {
+
-		rs, dgst, err := testutil.CreateRandomTarFile()
+	platformSpec := &v1.Platform{
+		Architecture: "atari2600",
+		OS:           "CP/M",
+	}
+
+	mfstDescriptors := []distribution.Descriptor{
+		createOciManifestDescriptor(t, testname, mfst, platformSpec),
+	}
+
+	imageIndex, err := ociIndexFromDesriptorsWithMediaType(mfstDescriptors, indexMediaType)
 	if err != nil {
-			t.Fatalf("%s: unexpected error generating test layer file", testname)
+		t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
 	}
 
-		wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
+	_, err = ms.Put(ctx, imageIndex)
-		if err != nil {
+	if err == nil {
-			t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
+		t.Fatalf("%s: expected error putting image index without child manifests in the registry: %v", testname, err)
 	}
 
-		if _, err := io.Copy(wr, rs); err != nil {
+	// Test for proper handling of SchemaVersion for the image
-			t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
-		}
-
-		if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
-			t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
-		}
-
-		if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
-			t.Fatalf("%s unexpected error appending references: %v", testname, err)
-		}
-	}
-
-	mfst, err := builder.Build(ctx)
-	if err != nil {
-		t.Fatalf("%s: unexpected error generating manifest: %v", testname, err)
-	}
-
-	// before putting the manifest test for proper handling of SchemaVersion
 
 	if mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion != 2 {
 		t.Fatalf("%s: unexpected error generating default version for oci manifest", testname)
@@ -379,22 +370,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
 		}
 	}
 
-	// Also create an image index that contains the manifest
+	// We can now push the index
-
-	descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest)
-	if err != nil {
-		t.Fatalf("%s: unexpected error getting manifest descriptor", testname)
-	}
-	descriptor.MediaType = v1.MediaTypeImageManifest
-	descriptor.Platform = &v1.Platform{
-		Architecture: "atari2600",
-		OS:           "CP/M",
-	}
-
-	imageIndex, err := ociIndexFromDesriptorsWithMediaType([]distribution.Descriptor{descriptor}, indexMediaType)
-	if err != nil {
-		t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
-	}
 
 	var indexDigest digest.Digest
 	if indexDigest, err = ms.Put(ctx, imageIndex); err != nil {
@@ -456,6 +432,244 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo
 	}
 }
 
+func TestIndexManifestStorageWithoutImageCheck(t *testing.T) {
+	imageMediaType := v1.MediaTypeImageManifest
+	indexMediaType := v1.MediaTypeImageIndex
+
+	repoName, _ := reference.WithName("foo/bar")
+	env := newManifestStoreTestEnv(t, repoName, "thetag",
+		BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
+		EnableDelete, EnableRedirect)
+
+	ctx := context.Background()
+	ms, err := env.repository.Manifests(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Build a manifest and store its layers in the registry
+
+	blobStore := env.repository.Blobs(ctx)
+	manifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
+	if err != nil {
+		t.Fatalf("unexpected error generating random image: %v", err)
+	}
+
+	// create an image index
+
+	ociPlatformSpec := &v1.Platform{
+		Architecture: "atari2600",
+		OS:           "CP/M",
+	}
+
+	ociManifestDescriptors := []distribution.Descriptor{
+		createOciManifestDescriptor(t, t.Name(), manifest, ociPlatformSpec),
+	}
+
+	imageIndex, err := ociIndexFromDesriptorsWithMediaType(ociManifestDescriptors, indexMediaType)
+	if err != nil {
+		t.Fatalf("unexpected error creating image index: %v", err)
+	}
+
+	// We should be able to put the index without having put the image
+
+	_, err = ms.Put(ctx, imageIndex)
+	if err != nil {
+		t.Fatalf("unexpected error putting sparse OCI image index: %v", err)
+	}
+
+	// same for a manifest list
+
+	listPlatformSpec := &manifestlist.PlatformSpec{
+		Architecture: "atari2600",
+		OS:           "CP/M",
+	}
+
+	listManifestDescriptors := []manifestlist.ManifestDescriptor{
+		createManifestListDescriptor(t, t.Name(), manifest, listPlatformSpec),
+	}
+
+	list, err := manifestlist.FromDescriptors(listManifestDescriptors)
+	if err != nil {
+		t.Fatalf("unexpected error creating manifest list: %v", err)
+	}
+
+	// We should be able to put the list without having put the image
+
+	_, err = ms.Put(ctx, list)
+	if err != nil {
+		t.Fatalf("unexpected error putting sparse manifest list: %v", err)
+	}
+}
+
+func TestIndexManifestStorageWithSelectivePlatforms(t *testing.T) {
+	imageMediaType := v1.MediaTypeImageManifest
+	indexMediaType := v1.MediaTypeImageIndex
+
+	repoName, _ := reference.WithName("foo/bar")
+	env := newManifestStoreTestEnv(t, repoName, "thetag",
+		BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
+		EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist,
+		AddValidateImageIndexImagesExistPlatform("amd64", "linux"))
+
+	ctx := context.Background()
+	ms, err := env.repository.Manifests(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Build a manifests their layers in the registry
+
+	blobStore := env.repository.Blobs(ctx)
+	amdManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
+	if err != nil {
+		t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
+	}
+	armManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
+	if err != nil {
+		t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
+	}
+	atariManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
+	if err != nil {
+		t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
+	}
+
+	// create an image index
+
+	amdPlatformSpec := &v1.Platform{
+		Architecture: "amd64",
+		OS:           "linux",
+	}
+	armPlatformSpec := &v1.Platform{
+		Architecture: "arm",
+		OS:           "plan9",
+	}
+	atariPlatformSpec := &v1.Platform{
+		Architecture: "atari2600",
+		OS:           "CP/M",
+	}
+
+	manifestDescriptors := []distribution.Descriptor{
+		createOciManifestDescriptor(t, t.Name(), amdManifest, amdPlatformSpec),
+		createOciManifestDescriptor(t, t.Name(), armManifest, armPlatformSpec),
+		createOciManifestDescriptor(t, t.Name(), atariManifest, atariPlatformSpec),
+	}
+
+	imageIndex, err := ociIndexFromDesriptorsWithMediaType(manifestDescriptors, indexMediaType)
+	if err != nil {
+		t.Fatalf("unexpected error creating image index: %v", err)
+	}
+
+	// Test we can't push with no image manifests existing in the registry
+
+	_, err = ms.Put(ctx, imageIndex)
+	if err == nil {
+		t.Fatalf("expected error putting image index without existing images: %v", err)
+	}
+
+	// Test we can't push with a manifest but not the right one
+
+	_, err = ms.Put(ctx, atariManifest)
+	if err != nil {
+		t.Fatalf("unexpected error putting manifest: %v", err)
+	}
+
+	_, err = ms.Put(ctx, imageIndex)
+	if err == nil {
+		t.Fatalf("expected error putting image index without correct existing images: %v", err)
+	}
+
+	// Test we can push with the right manifest
+
+	_, err = ms.Put(ctx, amdManifest)
+	if err != nil {
+		t.Fatalf("unexpected error putting manifest: %v", err)
+	}
+
+	_, err = ms.Put(ctx, imageIndex)
+	if err != nil {
+		t.Fatalf("unexpected error putting image index: %v", err)
+	}
+}
+
+// createRandomImage builds an image manifest and store it and its layers in the registry
+func createRandomImage(t *testing.T, testname string, imageMediaType string, blobStore distribution.BlobStore) (distribution.Manifest, error) {
+	builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
+	err := builder.(*ocischema.Builder).SetMediaType(imageMediaType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+
+	// Add some layers
+	for i := 0; i < 2; i++ {
+		rs, dgst, err := testutil.CreateRandomTarFile()
+		if err != nil {
+			t.Fatalf("%s: unexpected error generating test layer file", testname)
+		}
+
+		wr, err := blobStore.Create(ctx)
+		if err != nil {
+			t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
+		}
+
+		if _, err := io.Copy(wr, rs); err != nil {
+			t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
+		}
+
+		if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
+			t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
+		}
+
+		if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
+			t.Fatalf("%s unexpected error appending references: %v", testname, err)
+		}
+	}
+
+	return builder.Build(ctx)
+}
+
+// createOciManifestDescriptor builds a manifest descriptor from a manifest and a platform descriptor
+func createOciManifestDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *v1.Platform) distribution.Descriptor {
+	manifestMediaType, manifestPayload, err := manifest.Payload()
+	if err != nil {
+		t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
+	}
+	manifestDigest := digest.FromBytes(manifestPayload)
+
+	return distribution.Descriptor{
+		Digest:    manifestDigest,
+		Size:      int64(len(manifestPayload)),
+		MediaType: manifestMediaType,
+		Platform: &v1.Platform{
+			Architecture: platformSpec.Architecture,
+			OS:           platformSpec.OS,
+		},
+	}
+}
+
+// createManifestListDescriptor builds a manifest descriptor from a manifest and a platform descriptor
+func createManifestListDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *manifestlist.PlatformSpec) manifestlist.ManifestDescriptor {
+	manifestMediaType, manifestPayload, err := manifest.Payload()
+	if err != nil {
+		t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
+	}
+	manifestDigest := digest.FromBytes(manifestPayload)
+
+	return manifestlist.ManifestDescriptor{
+		Descriptor: distribution.Descriptor{
+			Digest:    manifestDigest,
+			Size:      int64(len(manifestPayload)),
+			MediaType: manifestMediaType,
+		},
+		Platform: manifestlist.PlatformSpec{
+			Architecture: platformSpec.Architecture,
+			OS:           platformSpec.OS,
+		},
+	}
+}
+
 // TestLinkPathFuncs ensures that the link path functions behavior are locked
 // down and implemented as expected.
 func TestLinkPathFuncs(t *testing.T) {

registry/storage/registry.go

@@ -26,8 +26,11 @@ type registry struct {
 	tagLookupConcurrencyLimit    int
 	resumableDigestEnabled       bool
 	blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory
-	manifestURLs                 manifestURLs
 	driver                       storagedriver.StorageDriver
+
+	// Validation
+	manifestURLs         manifestURLs
+	validateImageIndexes validateImageIndexes
 }
 
 // manifestURLs holds regular expressions for controlling manifest URL whitelisting
@@ -36,6 +39,20 @@ type manifestURLs struct {
 	deny  *regexp.Regexp
 }
 
+// validateImageIndexImages holds configuration for validation of image indexes
+type validateImageIndexes struct {
+	// exist can be used to disable checking that platform images exist entirely. Default true.
+	imagesExist bool
+	// platforms can be used to only validate the existence of images for a set of platforms. The empty array means validate all platforms.
+	imagePlatforms []platform
+}
+
+// platform represents a platform to validate exists in the
+type platform struct {
+	architecture string
+	os           string
+}
+
 // RegistryOption is the type used for functional options for NewRegistry.
 type RegistryOption func(*registry) error
 
@@ -83,6 +100,28 @@ func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption {
 	}
 }
 
+// EnableValidateImageIndexImagesExist is a functional option for NewRegistry. It enables
+// validation that references exist before an image index is accepted.
+func EnableValidateImageIndexImagesExist(registry *registry) error {
+	registry.validateImageIndexes.imagesExist = true
+	return nil
+}
+
+// AddValidateImageIndexImagesExistPlatform returns a functional option for NewRegistry.
+// It adds a platform to check for existence before an image index is accepted.
+func AddValidateImageIndexImagesExistPlatform(architecture string, os string) RegistryOption {
+	return func(registry *registry) error {
+		registry.validateImageIndexes.imagePlatforms = append(
+			registry.validateImageIndexes.imagePlatforms,
+			platform{
+				architecture: architecture,
+				os:           os,
+			},
+		)
+		return nil
+	}
+}
+
 // BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the
 // factory to create BlobDescriptorServiceFactory middleware.
 func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption {
@@ -243,6 +282,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
 		ctx:                  ctx,
 		repository:           repo,
 		blobStore:            blobStore,
+		validateImageIndexes: repo.validateImageIndexes,
 	}
 
 	ms := &manifestStore{

registry/storage/schema2manifesthandler_test.go

@@ -18,7 +18,9 @@ func TestVerifyManifestForeignLayer(t *testing.T) {
 	inmemoryDriver := inmemory.New()
 	registry := createRegistry(t, inmemoryDriver,
 		ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
-		ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")))
+		ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
+		EnableValidateImageIndexImagesExist,
+	)
 	repo := makeRepository(t, registry, "test")
 	manifestService := makeManifestService(t, repo)
 
@@ -156,7 +158,9 @@ func TestVerifyManifestBlobLayerAndConfig(t *testing.T) {
 	inmemoryDriver := inmemory.New()
 	registry := createRegistry(t, inmemoryDriver,
 		ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
-		ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")))
+		ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")),
+		EnableValidateImageIndexImagesExist,
+	)
 
 	repo := makeRepository(t, registry, strings.ToLower(t.Name()))
 	manifestService := makeManifestService(t, repo)