2014-11-22 03:39:52 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
2015-01-14 20:02:43 +00:00
|
|
|
"bytes"
|
2017-08-11 22:31:16 +00:00
|
|
|
"context"
|
2023-03-31 10:35:30 +00:00
|
|
|
"encoding/json"
|
2015-01-17 02:24:07 +00:00
|
|
|
"io"
|
2014-11-22 03:39:52 +00:00
|
|
|
"reflect"
|
|
|
|
"testing"
|
|
|
|
|
2020-08-24 11:18:39 +00:00
|
|
|
"github.com/distribution/distribution/v3"
|
|
|
|
"github.com/distribution/distribution/v3/manifest"
|
2023-08-15 13:37:43 +00:00
|
|
|
"github.com/distribution/distribution/v3/manifest/manifestlist"
|
2020-08-24 11:18:39 +00:00
|
|
|
"github.com/distribution/distribution/v3/manifest/ocischema"
|
2023-08-28 10:44:46 +00:00
|
|
|
"github.com/distribution/distribution/v3/manifest/schema2"
|
2020-08-24 11:18:39 +00:00
|
|
|
"github.com/distribution/distribution/v3/registry/storage/cache/memory"
|
|
|
|
"github.com/distribution/distribution/v3/registry/storage/driver"
|
|
|
|
"github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
|
|
|
|
"github.com/distribution/distribution/v3/testutil"
|
2023-08-30 15:50:56 +00:00
|
|
|
"github.com/distribution/reference"
|
2016-12-17 00:28:34 +00:00
|
|
|
"github.com/opencontainers/go-digest"
|
2019-10-09 12:02:21 +00:00
|
|
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
2014-11-22 03:39:52 +00:00
|
|
|
)
|
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
type manifestStoreTestEnv struct {
|
|
|
|
ctx context.Context
|
|
|
|
driver driver.StorageDriver
|
2015-04-10 02:21:33 +00:00
|
|
|
registry distribution.Namespace
|
2015-02-26 23:47:04 +00:00
|
|
|
repository distribution.Repository
|
2015-12-15 22:35:23 +00:00
|
|
|
name reference.Named
|
2015-02-26 23:47:04 +00:00
|
|
|
tag string
|
|
|
|
}
|
|
|
|
|
2016-02-10 23:20:39 +00:00
|
|
|
func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, options ...RegistryOption) *manifestStoreTestEnv {
|
2015-02-09 22:44:58 +00:00
|
|
|
ctx := context.Background()
|
2022-11-26 13:11:08 +00:00
|
|
|
drvr := inmemory.New()
|
|
|
|
registry, err := NewRegistry(ctx, drvr, options...)
|
2015-08-18 17:56:27 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error creating registry: %v", err)
|
|
|
|
}
|
2015-07-24 06:16:27 +00:00
|
|
|
|
2015-02-13 21:59:50 +00:00
|
|
|
repo, err := registry.Repository(ctx, name)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error getting repo: %v", err)
|
|
|
|
}
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
return &manifestStoreTestEnv{
|
|
|
|
ctx: ctx,
|
2022-11-26 13:11:08 +00:00
|
|
|
driver: drvr,
|
2015-02-26 23:47:04 +00:00
|
|
|
registry: registry,
|
|
|
|
repository: repo,
|
|
|
|
name: name,
|
|
|
|
tag: tag,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestManifestStorage(t *testing.T) {
|
2023-08-15 13:37:43 +00:00
|
|
|
testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
|
2017-12-18 23:06:04 +00:00
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
func testManifestStorage(t *testing.T, options ...RegistryOption) {
|
2017-01-14 01:06:03 +00:00
|
|
|
repoName, _ := reference.WithName("foo/bar")
|
2016-02-10 23:20:39 +00:00
|
|
|
env := newManifestStoreTestEnv(t, repoName, "thetag", options...)
|
2015-06-15 17:39:34 +00:00
|
|
|
ctx := context.Background()
|
|
|
|
ms, err := env.repository.Manifests(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-02-26 23:47:04 +00:00
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
// Push a config, and reference it in the manifest
|
|
|
|
sampleConfig := []byte(`{
|
|
|
|
"architecture": "amd64",
|
|
|
|
"history": [
|
|
|
|
{
|
|
|
|
"created": "2015-10-31T22:22:54.690851953Z",
|
|
|
|
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
|
|
|
|
},
|
|
|
|
],
|
|
|
|
"rootfs": {
|
|
|
|
"diff_ids": [
|
|
|
|
"sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
|
|
|
|
],
|
|
|
|
"type": "layers"
|
|
|
|
}
|
|
|
|
}`)
|
|
|
|
|
|
|
|
// Build a manifest and store it and its layers in the registry
|
|
|
|
|
|
|
|
blobStore := env.repository.Blobs(ctx)
|
|
|
|
d, err := blobStore.Put(ctx, schema2.MediaTypeImageConfig, sampleConfig)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
builder := schema2.NewManifestBuilder(d, sampleConfig)
|
|
|
|
|
|
|
|
m := &schema2.Manifest{
|
2015-01-02 21:21:29 +00:00
|
|
|
Versioned: manifest.Versioned{
|
2023-08-28 10:44:46 +00:00
|
|
|
SchemaVersion: 2,
|
|
|
|
MediaType: schema2.MediaTypeManifest,
|
|
|
|
},
|
|
|
|
Config: distribution.Descriptor{
|
2023-08-28 10:44:49 +00:00
|
|
|
Digest: digest.FromBytes(sampleConfig),
|
|
|
|
Size: int64(len(sampleConfig)),
|
2023-08-28 10:44:46 +00:00
|
|
|
MediaType: schema2.MediaTypeImageConfig,
|
|
|
|
},
|
2023-08-28 10:44:49 +00:00
|
|
|
Layers: []distribution.Descriptor{},
|
2015-01-17 02:24:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Build up some test layers and add them to the manifest, saving the
|
|
|
|
// readseekers for upload later.
|
|
|
|
testLayers := map[digest.Digest]io.ReadSeeker{}
|
|
|
|
for i := 0; i < 2; i++ {
|
2019-02-05 00:01:04 +00:00
|
|
|
rs, dgst, err := testutil.CreateRandomTarFile()
|
2015-01-17 02:24:07 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error generating test layer file")
|
|
|
|
}
|
|
|
|
|
2019-02-05 00:01:04 +00:00
|
|
|
testLayers[dgst] = rs
|
2023-08-28 10:44:49 +00:00
|
|
|
layer := distribution.Descriptor{
|
|
|
|
Digest: dgst,
|
|
|
|
Size: 6323,
|
|
|
|
MediaType: schema2.MediaTypeLayer,
|
|
|
|
}
|
|
|
|
m.Layers = append(m.Layers, layer)
|
Refactor Blob Service API
This PR refactors the blob service API to be oriented around blob descriptors.
Identified by digests, blobs become an abstract entity that can be read and
written using a descriptor as a handle. This allows blobs to take many forms,
such as a ReadSeekCloser or a simple byte buffer, allowing blob oriented
operations to better integrate with blob agnostic APIs (such as the `io`
package). The error definitions are now better organized to reflect conditions
that can only be seen when interacting with the blob API.
The main benefit of this is to separate the much smaller metadata from large
file storage. Many benefits also follow from this. Reading and writing has
been separated into discrete services. Backend implementation is also
simplified, by reducing the amount of metadata that needs to be picked up to
simply serve a read. This also improves cacheability.
"Opening" a blob simply consists of an access check (Stat) and a path
calculation. Caching is greatly simplified and we've made the mapping of
provisional to canonical hashes a first-class concept. BlobDescriptorService
and BlobProvider can be combined in different ways to achieve varying effects.
Recommend Review Approach
-------------------------
This is a very large patch. While apologies are in order, we are getting a
considerable amount of refactoring. Most changes follow from the changes to
the root package (distribution), so start there. From there, the main changes
are in storage. Looking at (*repository).Blobs will help to understand the how
the linkedBlobStore is wired. One can explore the internals within and also
branch out into understanding the changes to the caching layer. Following the
descriptions below will also help to guide you.
To reduce the chances for regressions, it was critical that major changes to
unit tests were avoided. Where possible, they are left untouched and where
not, the spirit is hopefully captured. Pay particular attention to where
behavior may have changed.
Storage
-------
The primary changes to the `storage` package, other than the interface
updates, were to merge the layerstore and blobstore. Blob access is now
layered even further. The first layer, blobStore, exposes a global
`BlobStatter` and `BlobProvider`. Operations here provide a fast path for most
read operations that don't take access control into account. The
`linkedBlobStore` layers on top of the `blobStore`, providing repository-
scoped blob link management in the backend. The `linkedBlobStore` implements
the full `BlobStore` suite, providing access-controlled, repository-local blob
writers. The abstraction between the two is slightly broken in that
`linkedBlobStore` is the only channel under which one can write into the global
blob store. The `linkedBlobStore` also provides flexibility in that it can act
over different link sets depending on configuration. This allows us to use the
same code for signature links, manifest links and blob links. Eventually, we
will fully consolidate this storage.
The improved cache flow comes from the `linkedBlobStatter` component
of `linkedBlobStore`. Using a `cachedBlobStatter`, these combine together to
provide a simple cache hierarchy that should streamline access checks on read
and write operations, or at least provide a single path to optimize. The
metrics have been changed in a slightly incompatible way since the former
operations, Fetch and Exists, are no longer relevant.
The fileWriter and fileReader have been slightly modified to support the rest
of the changes. The most interesting is the removal of the `Stat` call from
`newFileReader`. This was the source of unnecessary round trips that were only
present to look up the size of the resulting reader. Now, one must simply pass
in the size, requiring the caller to decide whether or not the `Stat` call is
appropriate. In several cases, it turned out the caller already had the size
already. The `WriterAt` implementation has been removed from `fileWriter`,
since it is no longer required for `BlobWriter`, reducing the number of paths
which writes may take.
Cache
-----
Unfortunately, the `cache` package required a near full rewrite. It was pretty
mechanical in that the cache is oriented around the `BlobDescriptorService`
slightly modified to include the ability to set the values for individual
digests. While the implementation is oriented towards caching, it can act as a
primary store. Provisions are in place to have repository local metadata, in
addition to global metadata. Fallback is implemented as a part of the storage
package to maintain this flexibility.
One unfortunate side-effect is that caching is now repository-scoped, rather
than global. This should have little effect on performance but may increase
memory usage.
Handlers
--------
The `handlers` package has been updated to leverage the new API. For the most
part, the changes are superficial or mechanical based on the API changes. This
did expose a bug in the handling of provisional vs canonical digests that was
fixed in the unit tests.
Configuration
-------------
One user-facing change has been made to the configuration and is updated in
the associated documentation. The `layerinfo` cache parameter has been
deprecated by the `blobdescriptor` cache parameter. Both are equivalent and
configuration files should be backward compatible.
Notifications
-------------
Changes the `notification` package are simply to support the interface
changes.
Context
-------
A small change has been made to the tracing log-level. Traces have been moved
from "info" to "debug" level to reduce output when not needed.
Signed-off-by: Stephen J Day <stephen.day@docker.com>
2015-05-12 07:10:29 +00:00
|
|
|
}
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-01-17 02:24:07 +00:00
|
|
|
// Now, upload the layers that were missing!
|
|
|
|
for dgst, rs := range testLayers {
|
Refactor Blob Service API
This PR refactors the blob service API to be oriented around blob descriptors.
Identified by digests, blobs become an abstract entity that can be read and
written using a descriptor as a handle. This allows blobs to take many forms,
such as a ReadSeekCloser or a simple byte buffer, allowing blob oriented
operations to better integrate with blob agnostic APIs (such as the `io`
package). The error definitions are now better organized to reflect conditions
that can only be seen when interacting with the blob API.
The main benefit of this is to separate the much smaller metadata from large
file storage. Many benefits also follow from this. Reading and writing has
been separated into discrete services. Backend implementation is also
simplified, by reducing the amount of metadata that needs to be picked up to
simply serve a read. This also improves cacheability.
"Opening" a blob simply consists of an access check (Stat) and a path
calculation. Caching is greatly simplified and we've made the mapping of
provisional to canonical hashes a first-class concept. BlobDescriptorService
and BlobProvider can be combined in different ways to achieve varying effects.
Recommend Review Approach
-------------------------
This is a very large patch. While apologies are in order, we are getting a
considerable amount of refactoring. Most changes follow from the changes to
the root package (distribution), so start there. From there, the main changes
are in storage. Looking at (*repository).Blobs will help to understand the how
the linkedBlobStore is wired. One can explore the internals within and also
branch out into understanding the changes to the caching layer. Following the
descriptions below will also help to guide you.
To reduce the chances for regressions, it was critical that major changes to
unit tests were avoided. Where possible, they are left untouched and where
not, the spirit is hopefully captured. Pay particular attention to where
behavior may have changed.
Storage
-------
The primary changes to the `storage` package, other than the interface
updates, were to merge the layerstore and blobstore. Blob access is now
layered even further. The first layer, blobStore, exposes a global
`BlobStatter` and `BlobProvider`. Operations here provide a fast path for most
read operations that don't take access control into account. The
`linkedBlobStore` layers on top of the `blobStore`, providing repository-
scoped blob link management in the backend. The `linkedBlobStore` implements
the full `BlobStore` suite, providing access-controlled, repository-local blob
writers. The abstraction between the two is slightly broken in that
`linkedBlobStore` is the only channel under which one can write into the global
blob store. The `linkedBlobStore` also provides flexibility in that it can act
over different link sets depending on configuration. This allows us to use the
same code for signature links, manifest links and blob links. Eventually, we
will fully consolidate this storage.
The improved cache flow comes from the `linkedBlobStatter` component
of `linkedBlobStore`. Using a `cachedBlobStatter`, these combine together to
provide a simple cache hierarchy that should streamline access checks on read
and write operations, or at least provide a single path to optimize. The
metrics have been changed in a slightly incompatible way since the former
operations, Fetch and Exists, are no longer relevant.
The fileWriter and fileReader have been slightly modified to support the rest
of the changes. The most interesting is the removal of the `Stat` call from
`newFileReader`. This was the source of unnecessary round trips that were only
present to look up the size of the resulting reader. Now, one must simply pass
in the size, requiring the caller to decide whether or not the `Stat` call is
appropriate. In several cases, it turned out the caller already had the size
already. The `WriterAt` implementation has been removed from `fileWriter`,
since it is no longer required for `BlobWriter`, reducing the number of paths
which writes may take.
Cache
-----
Unfortunately, the `cache` package required a near full rewrite. It was pretty
mechanical in that the cache is oriented around the `BlobDescriptorService`
slightly modified to include the ability to set the values for individual
digests. While the implementation is oriented towards caching, it can act as a
primary store. Provisions are in place to have repository local metadata, in
addition to global metadata. Fallback is implemented as a part of the storage
package to maintain this flexibility.
One unfortunate side-effect is that caching is now repository-scoped, rather
than global. This should have little effect on performance but may increase
memory usage.
Handlers
--------
The `handlers` package has been updated to leverage the new API. For the most
part, the changes are superficial or mechanical based on the API changes. This
did expose a bug in the handling of provisional vs canonical digests that was
fixed in the unit tests.
Configuration
-------------
One user-facing change has been made to the configuration and is updated in
the associated documentation. The `layerinfo` cache parameter has been
deprecated by the `blobdescriptor` cache parameter. Both are equivalent and
configuration files should be backward compatible.
Notifications
-------------
Changes the `notification` package are simply to support the interface
changes.
Context
-------
A small change has been made to the tracing log-level. Traces have been moved
from "info" to "debug" level to reduce output when not needed.
Signed-off-by: Stephen J Day <stephen.day@docker.com>
2015-05-12 07:10:29 +00:00
|
|
|
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
|
2015-01-17 02:24:07 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error creating test upload: %v", err)
|
|
|
|
}
|
|
|
|
|
Refactor Blob Service API
This PR refactors the blob service API to be oriented around blob descriptors.
Identified by digests, blobs become an abstract entity that can be read and
written using a descriptor as a handle. This allows blobs to take many forms,
such as a ReadSeekCloser or a simple byte buffer, allowing blob oriented
operations to better integrate with blob agnostic APIs (such as the `io`
package). The error definitions are now better organized to reflect conditions
that can only be seen when interacting with the blob API.
The main benefit of this is to separate the much smaller metadata from large
file storage. Many benefits also follow from this. Reading and writing has
been separated into discrete services. Backend implementation is also
simplified, by reducing the amount of metadata that needs to be picked up to
simply serve a read. This also improves cacheability.
"Opening" a blob simply consists of an access check (Stat) and a path
calculation. Caching is greatly simplified and we've made the mapping of
provisional to canonical hashes a first-class concept. BlobDescriptorService
and BlobProvider can be combined in different ways to achieve varying effects.
Recommend Review Approach
-------------------------
This is a very large patch. While apologies are in order, we are getting a
considerable amount of refactoring. Most changes follow from the changes to
the root package (distribution), so start there. From there, the main changes
are in storage. Looking at (*repository).Blobs will help to understand the how
the linkedBlobStore is wired. One can explore the internals within and also
branch out into understanding the changes to the caching layer. Following the
descriptions below will also help to guide you.
To reduce the chances for regressions, it was critical that major changes to
unit tests were avoided. Where possible, they are left untouched and where
not, the spirit is hopefully captured. Pay particular attention to where
behavior may have changed.
Storage
-------
The primary changes to the `storage` package, other than the interface
updates, were to merge the layerstore and blobstore. Blob access is now
layered even further. The first layer, blobStore, exposes a global
`BlobStatter` and `BlobProvider`. Operations here provide a fast path for most
read operations that don't take access control into account. The
`linkedBlobStore` layers on top of the `blobStore`, providing repository-
scoped blob link management in the backend. The `linkedBlobStore` implements
the full `BlobStore` suite, providing access-controlled, repository-local blob
writers. The abstraction between the two is slightly broken in that
`linkedBlobStore` is the only channel under which one can write into the global
blob store. The `linkedBlobStore` also provides flexibility in that it can act
over different link sets depending on configuration. This allows us to use the
same code for signature links, manifest links and blob links. Eventually, we
will fully consolidate this storage.
The improved cache flow comes from the `linkedBlobStatter` component
of `linkedBlobStore`. Using a `cachedBlobStatter`, these combine together to
provide a simple cache hierarchy that should streamline access checks on read
and write operations, or at least provide a single path to optimize. The
metrics have been changed in a slightly incompatible way since the former
operations, Fetch and Exists, are no longer relevant.
The fileWriter and fileReader have been slightly modified to support the rest
of the changes. The most interesting is the removal of the `Stat` call from
`newFileReader`. This was the source of unnecessary round trips that were only
present to look up the size of the resulting reader. Now, one must simply pass
in the size, requiring the caller to decide whether or not the `Stat` call is
appropriate. In several cases, it turned out the caller already had the size
already. The `WriterAt` implementation has been removed from `fileWriter`,
since it is no longer required for `BlobWriter`, reducing the number of paths
which writes may take.
Cache
-----
Unfortunately, the `cache` package required a near full rewrite. It was pretty
mechanical in that the cache is oriented around the `BlobDescriptorService`
slightly modified to include the ability to set the values for individual
digests. While the implementation is oriented towards caching, it can act as a
primary store. Provisions are in place to have repository local metadata, in
addition to global metadata. Fallback is implemented as a part of the storage
package to maintain this flexibility.
One unfortunate side-effect is that caching is now repository-scoped, rather
than global. This should have little effect on performance but may increase
memory usage.
Handlers
--------
The `handlers` package has been updated to leverage the new API. For the most
part, the changes are superficial or mechanical based on the API changes. This
did expose a bug in the handling of provisional vs canonical digests that was
fixed in the unit tests.
Configuration
-------------
One user-facing change has been made to the configuration and is updated in
the associated documentation. The `layerinfo` cache parameter has been
deprecated by the `blobdescriptor` cache parameter. Both are equivalent and
configuration files should be backward compatible.
Notifications
-------------
Changes the `notification` package are simply to support the interface
changes.
Context
-------
A small change has been made to the tracing log-level. Traces have been moved
from "info" to "debug" level to reduce output when not needed.
Signed-off-by: Stephen J Day <stephen.day@docker.com>
2015-05-12 07:10:29 +00:00
|
|
|
if _, err := io.Copy(wr, rs); err != nil {
|
2015-01-17 02:24:07 +00:00
|
|
|
t.Fatalf("unexpected error copying to upload: %v", err)
|
|
|
|
}
|
2014-11-22 03:39:52 +00:00
|
|
|
|
Refactor Blob Service API
This PR refactors the blob service API to be oriented around blob descriptors.
Identified by digests, blobs become an abstract entity that can be read and
written using a descriptor as a handle. This allows blobs to take many forms,
such as a ReadSeekCloser or a simple byte buffer, allowing blob oriented
operations to better integrate with blob agnostic APIs (such as the `io`
package). The error definitions are now better organized to reflect conditions
that can only be seen when interacting with the blob API.
The main benefit of this is to separate the much smaller metadata from large
file storage. Many benefits also follow from this. Reading and writing has
been separated into discrete services. Backend implementation is also
simplified, by reducing the amount of metadata that needs to be picked up to
simply serve a read. This also improves cacheability.
"Opening" a blob simply consists of an access check (Stat) and a path
calculation. Caching is greatly simplified and we've made the mapping of
provisional to canonical hashes a first-class concept. BlobDescriptorService
and BlobProvider can be combined in different ways to achieve varying effects.
Recommend Review Approach
-------------------------
This is a very large patch. While apologies are in order, we are getting a
considerable amount of refactoring. Most changes follow from the changes to
the root package (distribution), so start there. From there, the main changes
are in storage. Looking at (*repository).Blobs will help to understand the how
the linkedBlobStore is wired. One can explore the internals within and also
branch out into understanding the changes to the caching layer. Following the
descriptions below will also help to guide you.
To reduce the chances for regressions, it was critical that major changes to
unit tests were avoided. Where possible, they are left untouched and where
not, the spirit is hopefully captured. Pay particular attention to where
behavior may have changed.
Storage
-------
The primary changes to the `storage` package, other than the interface
updates, were to merge the layerstore and blobstore. Blob access is now
layered even further. The first layer, blobStore, exposes a global
`BlobStatter` and `BlobProvider`. Operations here provide a fast path for most
read operations that don't take access control into account. The
`linkedBlobStore` layers on top of the `blobStore`, providing repository-
scoped blob link management in the backend. The `linkedBlobStore` implements
the full `BlobStore` suite, providing access-controlled, repository-local blob
writers. The abstraction between the two is slightly broken in that
`linkedBlobStore` is the only channel under which one can write into the global
blob store. The `linkedBlobStore` also provides flexibility in that it can act
over different link sets depending on configuration. This allows us to use the
same code for signature links, manifest links and blob links. Eventually, we
will fully consolidate this storage.
The improved cache flow comes from the `linkedBlobStatter` component
of `linkedBlobStore`. Using a `cachedBlobStatter`, these combine together to
provide a simple cache hierarchy that should streamline access checks on read
and write operations, or at least provide a single path to optimize. The
metrics have been changed in a slightly incompatible way since the former
operations, Fetch and Exists, are no longer relevant.
The fileWriter and fileReader have been slightly modified to support the rest
of the changes. The most interesting is the removal of the `Stat` call from
`newFileReader`. This was the source of unnecessary round trips that were only
present to look up the size of the resulting reader. Now, one must simply pass
in the size, requiring the caller to decide whether or not the `Stat` call is
appropriate. In several cases, it turned out the caller already had the size
already. The `WriterAt` implementation has been removed from `fileWriter`,
since it is no longer required for `BlobWriter`, reducing the number of paths
which writes may take.
Cache
-----
Unfortunately, the `cache` package required a near full rewrite. It was pretty
mechanical in that the cache is oriented around the `BlobDescriptorService`
slightly modified to include the ability to set the values for individual
digests. While the implementation is oriented towards caching, it can act as a
primary store. Provisions are in place to have repository local metadata, in
addition to global metadata. Fallback is implemented as a part of the storage
package to maintain this flexibility.
One unfortunate side-effect is that caching is now repository-scoped, rather
than global. This should have little effect on performance but may increase
memory usage.
Handlers
--------
The `handlers` package has been updated to leverage the new API. For the most
part, the changes are superficial or mechanical based on the API changes. This
did expose a bug in the handling of provisional vs canonical digests that was
fixed in the unit tests.
Configuration
-------------
One user-facing change has been made to the configuration and is updated in
the associated documentation. The `layerinfo` cache parameter has been
deprecated by the `blobdescriptor` cache parameter. Both are equivalent and
configuration files should be backward compatible.
Notifications
-------------
Changes the `notification` package are simply to support the interface
changes.
Context
-------
A small change has been made to the tracing log-level. Traces have been moved
from "info" to "debug" level to reduce output when not needed.
Signed-off-by: Stephen J Day <stephen.day@docker.com>
2015-05-12 07:10:29 +00:00
|
|
|
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
|
2015-01-17 02:24:07 +00:00
|
|
|
t.Fatalf("unexpected error finishing upload: %v", err)
|
|
|
|
}
|
2023-11-18 06:50:40 +00:00
|
|
|
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: schema2.MediaTypeLayer}); err != nil {
|
|
|
|
t.Fatalf("unexpected error appending references: %v", err)
|
|
|
|
}
|
2023-08-28 10:44:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
sm, err := builder.Build(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error generating manifest: %v", repoName, err)
|
2015-01-17 02:24:07 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
var manifestDigest digest.Digest
|
|
|
|
if manifestDigest, err = ms.Put(ctx, sm); err != nil {
|
2014-11-22 03:39:52 +00:00
|
|
|
t.Fatalf("unexpected error putting manifest: %v", err)
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
exists, err := ms.Exists(ctx, manifestDigest)
|
2014-11-22 03:39:52 +00:00
|
|
|
if err != nil {
|
2015-08-21 04:50:15 +00:00
|
|
|
t.Fatalf("unexpected error checking manifest existence: %#v", err)
|
2014-11-22 03:39:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if !exists {
|
|
|
|
t.Fatalf("manifest should exist")
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
fromStore, err := ms.Get(ctx, manifestDigest)
|
2014-11-22 03:39:52 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error fetching manifest: %v", err)
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
fetchedManifest, ok := fromStore.(*schema2.DeserializedManifest)
|
2015-08-21 04:50:15 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatalf("unexpected manifest type from signedstore")
|
|
|
|
}
|
2023-08-28 10:44:46 +00:00
|
|
|
_, pl, err := fetchedManifest.Payload()
|
2015-01-14 20:02:43 +00:00
|
|
|
if err != nil {
|
2023-08-28 10:44:46 +00:00
|
|
|
t.Fatalf("could not get manifest payload: %v", err)
|
2015-01-14 20:02:43 +00:00
|
|
|
}
|
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
// Now that we have a payload, take a moment to check that the manifest is
|
|
|
|
// return by the payload digest.
|
2015-08-21 04:50:15 +00:00
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
dgst := digest.FromBytes(pl)
|
2015-08-21 04:50:15 +00:00
|
|
|
exists, err = ms.Exists(ctx, dgst)
|
2015-02-26 23:47:04 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error checking manifest existence by digest: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !exists {
|
|
|
|
t.Fatalf("manifest %s should exist", dgst)
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
fetchedByDigest, err := ms.Get(ctx, dgst)
|
2015-02-26 23:47:04 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
byDigestManifest, ok := fetchedByDigest.(*schema2.DeserializedManifest)
|
2016-02-10 23:20:39 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatalf("unexpected manifest type from signedstore")
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
_, byDigestCanonical, err := byDigestManifest.Payload()
|
2015-01-14 20:02:43 +00:00
|
|
|
if err != nil {
|
2023-08-28 10:44:46 +00:00
|
|
|
t.Fatalf("could not get manifest payload: %v", err)
|
2015-01-14 20:02:43 +00:00
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
if !bytes.Equal(byDigestCanonical, pl) {
|
|
|
|
t.Fatalf("fetched manifest not equal: %q != %q", byDigestCanonical, pl)
|
2015-01-14 20:02:43 +00:00
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
fromStore, err = ms.Get(ctx, manifestDigest)
|
2015-01-14 20:02:43 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error fetching manifest: %v", err)
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
fetched, ok := fromStore.(*schema2.DeserializedManifest)
|
2015-08-21 04:50:15 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
_, receivedPL, err := fetched.Payload()
|
2015-08-21 04:50:15 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error getting payload %#v", err)
|
|
|
|
}
|
|
|
|
|
2023-08-28 10:44:46 +00:00
|
|
|
if !bytes.Equal(receivedPL, pl) {
|
2015-01-14 20:02:43 +00:00
|
|
|
t.Fatalf("payloads are not equal")
|
|
|
|
}
|
|
|
|
|
2015-05-27 17:52:22 +00:00
|
|
|
// Test deleting manifests
|
2015-08-21 04:50:15 +00:00
|
|
|
err = ms.Delete(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err != nil {
|
2015-02-26 23:47:04 +00:00
|
|
|
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
|
2015-01-14 20:02:43 +00:00
|
|
|
}
|
2015-05-27 17:52:22 +00:00
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
exists, err = ms.Exists(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Error querying manifest existence")
|
|
|
|
}
|
|
|
|
if exists {
|
|
|
|
t.Errorf("Deleted manifest should not exist")
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
deletedManifest, err := ms.Get(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Errorf("Unexpected success getting deleted manifest")
|
|
|
|
}
|
|
|
|
switch err.(type) {
|
|
|
|
case distribution.ErrManifestUnknownRevision:
|
|
|
|
break
|
|
|
|
default:
|
|
|
|
t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type())
|
|
|
|
}
|
|
|
|
|
|
|
|
if deletedManifest != nil {
|
|
|
|
t.Errorf("Deleted manifest get returned non-nil")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Re-upload should restore manifest to a good state
|
2015-08-21 04:50:15 +00:00
|
|
|
_, err = ms.Put(ctx, sm)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Errorf("Error re-uploading deleted manifest")
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
exists, err = ms.Exists(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Error querying manifest existence")
|
|
|
|
}
|
|
|
|
if !exists {
|
|
|
|
t.Errorf("Restored manifest should exist")
|
|
|
|
}
|
|
|
|
|
2015-08-21 04:50:15 +00:00
|
|
|
deletedManifest, err = ms.Get(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Errorf("Unexpected error getting manifest")
|
|
|
|
}
|
|
|
|
if deletedManifest == nil {
|
|
|
|
t.Errorf("Deleted manifest get returned non-nil")
|
|
|
|
}
|
|
|
|
|
2022-07-13 00:42:48 +00:00
|
|
|
r, err := NewRegistry(ctx, env.driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)), EnableRedirect)
|
2015-08-18 17:56:27 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error creating registry: %v", err)
|
|
|
|
}
|
2015-05-27 17:52:22 +00:00
|
|
|
repo, err := r.Repository(ctx, env.name)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error getting repo: %v", err)
|
|
|
|
}
|
|
|
|
ms, err = repo.Manifests(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2015-08-21 04:50:15 +00:00
|
|
|
err = ms.Delete(ctx, dgst)
|
2015-05-27 17:52:22 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Errorf("Unexpected success deleting while disabled")
|
|
|
|
}
|
2014-11-22 03:39:52 +00:00
|
|
|
}
|
2015-08-12 20:07:57 +00:00
|
|
|
|
2018-03-15 20:04:58 +00:00
|
|
|
func TestOCIManifestStorage(t *testing.T) {
|
|
|
|
testOCIManifestStorage(t, "includeMediaTypes=true", true)
|
|
|
|
testOCIManifestStorage(t, "includeMediaTypes=false", false)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes bool) {
|
|
|
|
var imageMediaType string
|
|
|
|
var indexMediaType string
|
|
|
|
if includeMediaTypes {
|
|
|
|
imageMediaType = v1.MediaTypeImageManifest
|
|
|
|
indexMediaType = v1.MediaTypeImageIndex
|
|
|
|
} else {
|
|
|
|
imageMediaType = ""
|
|
|
|
indexMediaType = ""
|
|
|
|
}
|
|
|
|
|
|
|
|
repoName, _ := reference.WithName("foo/bar")
|
|
|
|
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
2022-07-13 00:42:48 +00:00
|
|
|
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
2023-08-15 13:37:43 +00:00
|
|
|
EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist)
|
2018-03-15 20:04:58 +00:00
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
ms, err := env.repository.Manifests(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
// Build a manifest and store its layers in the registry
|
2018-03-15 20:04:58 +00:00
|
|
|
|
|
|
|
blobStore := env.repository.Blobs(ctx)
|
2023-08-15 13:37:43 +00:00
|
|
|
mfst, err := createRandomImage(t, testname, imageMediaType, blobStore)
|
2018-07-19 21:07:26 +00:00
|
|
|
if err != nil {
|
2023-08-15 13:37:43 +00:00
|
|
|
t.Fatalf("%s: unexpected error generating random image: %v", testname, err)
|
2018-07-19 21:07:26 +00:00
|
|
|
}
|
2018-03-15 20:04:58 +00:00
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
// create an image index
|
2018-03-15 20:04:58 +00:00
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
platformSpec := &v1.Platform{
|
|
|
|
Architecture: "atari2600",
|
|
|
|
OS: "CP/M",
|
|
|
|
}
|
2018-03-15 20:04:58 +00:00
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
mfstDescriptors := []distribution.Descriptor{
|
|
|
|
createOciManifestDescriptor(t, testname, mfst, platformSpec),
|
2018-03-15 20:04:58 +00:00
|
|
|
}
|
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
imageIndex, err := ociIndexFromDesriptorsWithMediaType(mfstDescriptors, indexMediaType)
|
2018-03-15 20:04:58 +00:00
|
|
|
if err != nil {
|
2023-08-15 13:37:43 +00:00
|
|
|
t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, imageIndex)
|
|
|
|
if err == nil {
|
|
|
|
t.Fatalf("%s: expected error putting image index without child manifests in the registry: %v", testname, err)
|
2018-03-15 20:04:58 +00:00
|
|
|
}
|
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
// Test for proper handling of SchemaVersion for the image
|
2018-07-23 22:03:17 +00:00
|
|
|
|
2022-11-26 13:11:08 +00:00
|
|
|
if mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion != 2 {
|
2018-07-23 22:03:17 +00:00
|
|
|
t.Fatalf("%s: unexpected error generating default version for oci manifest", testname)
|
|
|
|
}
|
2022-11-26 13:11:08 +00:00
|
|
|
mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion = 0
|
2018-07-23 22:03:17 +00:00
|
|
|
|
2018-03-15 20:04:58 +00:00
|
|
|
var manifestDigest digest.Digest
|
2022-11-26 13:11:08 +00:00
|
|
|
if manifestDigest, err = ms.Put(ctx, mfst); err != nil {
|
2018-07-23 22:03:17 +00:00
|
|
|
if err.Error() != "unrecognized manifest schema version 0" {
|
|
|
|
t.Fatalf("%s: unexpected error putting manifest: %v", testname, err)
|
|
|
|
}
|
2022-11-26 13:11:08 +00:00
|
|
|
mfst.(*ocischema.DeserializedManifest).Manifest.SchemaVersion = 2
|
|
|
|
if manifestDigest, err = ms.Put(ctx, mfst); err != nil {
|
2018-07-23 22:03:17 +00:00
|
|
|
t.Fatalf("%s: unexpected error putting manifest: %v", testname, err)
|
|
|
|
}
|
2018-03-15 20:04:58 +00:00
|
|
|
}
|
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
// We can now push the index
|
2018-03-15 20:04:58 +00:00
|
|
|
|
|
|
|
var indexDigest digest.Digest
|
|
|
|
if indexDigest, err = ms.Put(ctx, imageIndex); err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error putting image index: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now check that we can retrieve the manifest
|
|
|
|
|
|
|
|
fromStore, err := ms.Get(ctx, manifestDigest)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error fetching manifest: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fetchedManifest, ok := fromStore.(*ocischema.DeserializedManifest)
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("%s: unexpected type for fetched manifest", testname)
|
|
|
|
}
|
|
|
|
|
|
|
|
if fetchedManifest.MediaType != imageMediaType {
|
|
|
|
t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType)
|
|
|
|
}
|
|
|
|
|
2018-07-20 00:41:31 +00:00
|
|
|
if fetchedManifest.SchemaVersion != ocischema.SchemaVersion.SchemaVersion {
|
|
|
|
t.Fatalf("%s: unexpected schema version for result, %d", testname, fetchedManifest.SchemaVersion)
|
|
|
|
}
|
|
|
|
|
2018-03-15 20:04:58 +00:00
|
|
|
payloadMediaType, _, err := fromStore.Payload()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: error getting payload %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if payloadMediaType != v1.MediaTypeImageManifest {
|
|
|
|
t.Fatalf("%s: unexpected MediaType for manifest payload, %s", testname, payloadMediaType)
|
|
|
|
}
|
|
|
|
|
|
|
|
// and the image index
|
|
|
|
|
|
|
|
fromStore, err = ms.Get(ctx, indexDigest)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error fetching image index: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
2023-03-31 10:35:30 +00:00
|
|
|
fetchedIndex, ok := fromStore.(*ocischema.DeserializedImageIndex)
|
2018-03-15 20:04:58 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatalf("%s: unexpected type for fetched manifest", testname)
|
|
|
|
}
|
|
|
|
|
|
|
|
if fetchedIndex.MediaType != indexMediaType {
|
2022-03-27 10:04:19 +00:00
|
|
|
t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedIndex.MediaType)
|
2018-03-15 20:04:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
payloadMediaType, _, err = fromStore.Payload()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: error getting payload %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if payloadMediaType != v1.MediaTypeImageIndex {
|
|
|
|
t.Fatalf("%s: unexpected MediaType for index payload, %s", testname, payloadMediaType)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-15 13:37:43 +00:00
|
|
|
func TestIndexManifestStorageWithoutImageCheck(t *testing.T) {
|
|
|
|
imageMediaType := v1.MediaTypeImageManifest
|
|
|
|
indexMediaType := v1.MediaTypeImageIndex
|
|
|
|
|
|
|
|
repoName, _ := reference.WithName("foo/bar")
|
|
|
|
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
|
|
|
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
|
|
|
EnableDelete, EnableRedirect)
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
ms, err := env.repository.Manifests(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Build a manifest and store its layers in the registry
|
|
|
|
|
|
|
|
blobStore := env.repository.Blobs(ctx)
|
|
|
|
manifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error generating random image: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// create an image index
|
|
|
|
|
|
|
|
ociPlatformSpec := &v1.Platform{
|
|
|
|
Architecture: "atari2600",
|
|
|
|
OS: "CP/M",
|
|
|
|
}
|
|
|
|
|
|
|
|
ociManifestDescriptors := []distribution.Descriptor{
|
|
|
|
createOciManifestDescriptor(t, t.Name(), manifest, ociPlatformSpec),
|
|
|
|
}
|
|
|
|
|
|
|
|
imageIndex, err := ociIndexFromDesriptorsWithMediaType(ociManifestDescriptors, indexMediaType)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error creating image index: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// We should be able to put the index without having put the image
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, imageIndex)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error putting sparse OCI image index: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// same for a manifest list
|
|
|
|
|
|
|
|
listPlatformSpec := &manifestlist.PlatformSpec{
|
|
|
|
Architecture: "atari2600",
|
|
|
|
OS: "CP/M",
|
|
|
|
}
|
|
|
|
|
|
|
|
listManifestDescriptors := []manifestlist.ManifestDescriptor{
|
|
|
|
createManifestListDescriptor(t, t.Name(), manifest, listPlatformSpec),
|
|
|
|
}
|
|
|
|
|
|
|
|
list, err := manifestlist.FromDescriptors(listManifestDescriptors)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error creating manifest list: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// We should be able to put the list without having put the image
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, list)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error putting sparse manifest list: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestIndexManifestStorageWithSelectivePlatforms(t *testing.T) {
|
|
|
|
imageMediaType := v1.MediaTypeImageManifest
|
|
|
|
indexMediaType := v1.MediaTypeImageIndex
|
|
|
|
|
|
|
|
repoName, _ := reference.WithName("foo/bar")
|
|
|
|
env := newManifestStoreTestEnv(t, repoName, "thetag",
|
|
|
|
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider(memory.UnlimitedSize)),
|
|
|
|
EnableDelete, EnableRedirect, EnableValidateImageIndexImagesExist,
|
|
|
|
AddValidateImageIndexImagesExistPlatform("amd64", "linux"))
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
ms, err := env.repository.Manifests(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Build a manifests their layers in the registry
|
|
|
|
|
|
|
|
blobStore := env.repository.Blobs(ctx)
|
|
|
|
amdManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
|
|
|
}
|
|
|
|
armManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
|
|
|
}
|
|
|
|
atariManifest, err := createRandomImage(t, t.Name(), imageMediaType, blobStore)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error generating random image: %v", t.Name(), err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// create an image index
|
|
|
|
|
|
|
|
amdPlatformSpec := &v1.Platform{
|
|
|
|
Architecture: "amd64",
|
|
|
|
OS: "linux",
|
|
|
|
}
|
|
|
|
armPlatformSpec := &v1.Platform{
|
|
|
|
Architecture: "arm",
|
|
|
|
OS: "plan9",
|
|
|
|
}
|
|
|
|
atariPlatformSpec := &v1.Platform{
|
|
|
|
Architecture: "atari2600",
|
|
|
|
OS: "CP/M",
|
|
|
|
}
|
|
|
|
|
|
|
|
manifestDescriptors := []distribution.Descriptor{
|
|
|
|
createOciManifestDescriptor(t, t.Name(), amdManifest, amdPlatformSpec),
|
|
|
|
createOciManifestDescriptor(t, t.Name(), armManifest, armPlatformSpec),
|
|
|
|
createOciManifestDescriptor(t, t.Name(), atariManifest, atariPlatformSpec),
|
|
|
|
}
|
|
|
|
|
|
|
|
imageIndex, err := ociIndexFromDesriptorsWithMediaType(manifestDescriptors, indexMediaType)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error creating image index: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test we can't push with no image manifests existing in the registry
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, imageIndex)
|
|
|
|
if err == nil {
|
|
|
|
t.Fatalf("expected error putting image index without existing images: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test we can't push with a manifest but not the right one
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, atariManifest)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error putting manifest: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, imageIndex)
|
|
|
|
if err == nil {
|
|
|
|
t.Fatalf("expected error putting image index without correct existing images: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test we can push with the right manifest
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, amdManifest)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error putting manifest: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = ms.Put(ctx, imageIndex)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error putting image index: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// createRandomImage builds an image manifest and store it and its layers in the registry
|
|
|
|
func createRandomImage(t *testing.T, testname string, imageMediaType string, blobStore distribution.BlobStore) (distribution.Manifest, error) {
|
|
|
|
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
|
2023-05-02 16:09:48 +00:00
|
|
|
err := builder.SetMediaType(imageMediaType)
|
2023-08-15 13:37:43 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
|
|
|
|
// Add some layers
|
|
|
|
for i := 0; i < 2; i++ {
|
|
|
|
rs, dgst, err := testutil.CreateRandomTarFile()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error generating test layer file", testname)
|
|
|
|
}
|
|
|
|
|
|
|
|
wr, err := blobStore.Create(ctx)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := io.Copy(wr, rs); err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := builder.AppendReference(distribution.Descriptor{Digest: dgst, MediaType: v1.MediaTypeImageLayer}); err != nil {
|
|
|
|
t.Fatalf("%s unexpected error appending references: %v", testname, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return builder.Build(ctx)
|
|
|
|
}
|
|
|
|
|
|
|
|
// createOciManifestDescriptor builds a manifest descriptor from a manifest and a platform descriptor
|
|
|
|
func createOciManifestDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *v1.Platform) distribution.Descriptor {
|
|
|
|
manifestMediaType, manifestPayload, err := manifest.Payload()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
|
|
|
|
}
|
|
|
|
manifestDigest := digest.FromBytes(manifestPayload)
|
|
|
|
|
|
|
|
return distribution.Descriptor{
|
|
|
|
Digest: manifestDigest,
|
|
|
|
Size: int64(len(manifestPayload)),
|
|
|
|
MediaType: manifestMediaType,
|
|
|
|
Platform: &v1.Platform{
|
|
|
|
Architecture: platformSpec.Architecture,
|
|
|
|
OS: platformSpec.OS,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// createManifestListDescriptor builds a manifest descriptor from a manifest and a platform descriptor
|
|
|
|
func createManifestListDescriptor(t *testing.T, testname string, manifest distribution.Manifest, platformSpec *manifestlist.PlatformSpec) manifestlist.ManifestDescriptor {
|
|
|
|
manifestMediaType, manifestPayload, err := manifest.Payload()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%s: unexpected error getting manifest payload: %v", testname, err)
|
|
|
|
}
|
|
|
|
manifestDigest := digest.FromBytes(manifestPayload)
|
|
|
|
|
|
|
|
return manifestlist.ManifestDescriptor{
|
|
|
|
Descriptor: distribution.Descriptor{
|
|
|
|
Digest: manifestDigest,
|
|
|
|
Size: int64(len(manifestPayload)),
|
|
|
|
MediaType: manifestMediaType,
|
|
|
|
},
|
|
|
|
Platform: manifestlist.PlatformSpec{
|
|
|
|
Architecture: platformSpec.Architecture,
|
|
|
|
OS: platformSpec.OS,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-12 20:07:57 +00:00
|
|
|
// TestLinkPathFuncs ensures that the link path functions behavior are locked
|
|
|
|
// down and implemented as expected.
|
|
|
|
func TestLinkPathFuncs(t *testing.T) {
|
|
|
|
for _, testcase := range []struct {
|
|
|
|
repo string
|
|
|
|
digest digest.Digest
|
|
|
|
linkPathFn linkPathFunc
|
|
|
|
expected string
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
repo: "foo/bar",
|
2015-12-02 23:57:47 +00:00
|
|
|
digest: "sha256:deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
2015-08-12 20:07:57 +00:00
|
|
|
linkPathFn: blobLinkPath,
|
2015-12-02 23:57:47 +00:00
|
|
|
expected: "/docker/registry/v2/repositories/foo/bar/_layers/sha256/deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/link",
|
2015-08-12 20:07:57 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
repo: "foo/bar",
|
2015-12-02 23:57:47 +00:00
|
|
|
digest: "sha256:deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
2015-08-12 20:07:57 +00:00
|
|
|
linkPathFn: manifestRevisionLinkPath,
|
2015-12-02 23:57:47 +00:00
|
|
|
expected: "/docker/registry/v2/repositories/foo/bar/_manifests/revisions/sha256/deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/link",
|
2015-08-12 20:07:57 +00:00
|
|
|
},
|
|
|
|
} {
|
2015-08-18 01:51:05 +00:00
|
|
|
p, err := testcase.linkPathFn(testcase.repo, testcase.digest)
|
2015-08-12 20:07:57 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("unexpected error calling linkPathFn(pm, %q, %q): %v", testcase.repo, testcase.digest, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if p != testcase.expected {
|
|
|
|
t.Fatalf("incorrect path returned: %q != %q", p, testcase.expected)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-03-31 10:35:30 +00:00
|
|
|
|
2023-06-01 10:38:31 +00:00
|
|
|
func ociIndexFromDesriptorsWithMediaType(descriptors []distribution.Descriptor, mediaType string) (*ocischema.DeserializedImageIndex, error) {
|
2023-03-31 13:01:15 +00:00
|
|
|
manifest, err := ocischema.FromDescriptors(descriptors, nil)
|
2023-03-31 10:35:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
manifest.ImageIndex.MediaType = mediaType
|
|
|
|
|
|
|
|
rawManifest, err := json.Marshal(manifest.ImageIndex)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var d ocischema.DeserializedImageIndex
|
|
|
|
if err := d.UnmarshalJSON(rawManifest); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &d, nil
|
|
|
|
}
|