2015-02-12 00:49:49 +00:00
|
|
|
package distribution
|
2014-11-18 00:29:42 +00:00
|
|
|
|
|
|
|
import (
|
2015-02-13 21:59:50 +00:00
|
|
|
"io"
|
2015-03-03 16:57:52 +00:00
|
|
|
"net/http"
|
2015-02-13 21:59:50 +00:00
|
|
|
"time"
|
|
|
|
|
2014-12-24 00:01:38 +00:00
|
|
|
"github.com/docker/distribution/digest"
|
2015-01-02 21:21:29 +00:00
|
|
|
"github.com/docker/distribution/manifest"
|
2015-02-09 22:44:58 +00:00
|
|
|
"golang.org/x/net/context"
|
2014-11-18 00:29:42 +00:00
|
|
|
)
|
|
|
|
|
2015-01-17 02:24:07 +00:00
|
|
|
// Registry represents a collection of repositories, addressable by name.
|
|
|
|
type Registry interface {
|
|
|
|
// Repository should return a reference to the named repository. The
|
|
|
|
// registry may or may not have the repository but should always return a
|
|
|
|
// reference.
|
2015-02-13 21:59:50 +00:00
|
|
|
Repository(ctx context.Context, name string) (Repository, error)
|
2014-11-18 00:29:42 +00:00
|
|
|
}
|
|
|
|
|
2015-01-17 02:24:07 +00:00
|
|
|
// Repository is a named collection of manifests and layers.
|
|
|
|
type Repository interface {
|
|
|
|
// Name returns the name of the repository.
|
|
|
|
Name() string
|
|
|
|
|
|
|
|
// Manifests returns a reference to this repository's manifest service.
|
|
|
|
Manifests() ManifestService
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-01-17 02:24:07 +00:00
|
|
|
// Layers returns a reference to this repository's layers service.
|
|
|
|
Layers() LayerService
|
2015-03-04 20:32:22 +00:00
|
|
|
|
|
|
|
// Signatures returns a reference to this repository's signatures service.
|
|
|
|
Signatures() SignatureService
|
2014-11-22 03:39:52 +00:00
|
|
|
}
|
|
|
|
|
2015-04-01 23:41:33 +00:00
|
|
|
// TODO(stevvooe): Must add close methods to all these. May want to change the
|
|
|
|
// way instances are created to better reflect internal dependency
|
|
|
|
// relationships.
|
|
|
|
|
2014-11-22 03:39:52 +00:00
|
|
|
// ManifestService provides operations on image manifests.
|
|
|
|
type ManifestService interface {
|
2015-01-14 20:02:43 +00:00
|
|
|
// Exists returns true if the manifest exists.
|
2015-02-26 23:47:04 +00:00
|
|
|
Exists(dgst digest.Digest) (bool, error)
|
|
|
|
|
|
|
|
// Get retrieves the identified by the digest, if it exists.
|
|
|
|
Get(dgst digest.Digest) (*manifest.SignedManifest, error)
|
|
|
|
|
|
|
|
// Delete removes the manifest, if it exists.
|
|
|
|
Delete(dgst digest.Digest) error
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
// Put creates or updates the manifest.
|
|
|
|
Put(manifest *manifest.SignedManifest) error
|
|
|
|
|
|
|
|
// TODO(stevvooe): The methods after this message should be moved to a
|
|
|
|
// discrete TagService, per active proposals.
|
|
|
|
|
|
|
|
// Tags lists the tags under the named repository.
|
|
|
|
Tags() ([]string, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
// ExistsByTag returns true if the manifest exists.
|
|
|
|
ExistsByTag(tag string) (bool, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
|
2015-02-26 23:47:04 +00:00
|
|
|
// GetByTag retrieves the named manifest, if it exists.
|
|
|
|
GetByTag(tag string) (*manifest.SignedManifest, error)
|
2015-01-17 02:24:07 +00:00
|
|
|
|
|
|
|
// TODO(stevvooe): There are several changes that need to be done to this
|
|
|
|
// interface:
|
|
|
|
//
|
2015-02-26 23:47:04 +00:00
|
|
|
// 1. Allow explicit tagging with Tag(digest digest.Digest, tag string)
|
|
|
|
// 2. Support reading tags with a re-entrant reader to avoid large
|
2015-01-17 02:24:07 +00:00
|
|
|
// allocations in the registry.
|
2015-02-26 23:47:04 +00:00
|
|
|
// 3. Long-term: Provide All() method that lets one scroll through all of
|
2015-01-17 02:24:07 +00:00
|
|
|
// the manifest entries.
|
2015-02-26 23:47:04 +00:00
|
|
|
// 4. Long-term: break out concept of signing from manifests. This is
|
2015-01-17 02:24:07 +00:00
|
|
|
// really a part of the distribution sprint.
|
2015-02-26 23:47:04 +00:00
|
|
|
// 5. Long-term: Manifest should be an interface. This code shouldn't
|
2015-01-17 02:24:07 +00:00
|
|
|
// really be concerned with the storage format.
|
2014-11-22 03:39:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// LayerService provides operations on layer files in a backend storage.
|
|
|
|
type LayerService interface {
|
|
|
|
// Exists returns true if the layer exists.
|
2015-01-17 02:24:07 +00:00
|
|
|
Exists(digest digest.Digest) (bool, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
|
|
|
|
// Fetch the layer identifed by TarSum.
|
2015-01-17 02:24:07 +00:00
|
|
|
Fetch(digest digest.Digest) (Layer, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
|
|
|
|
// Upload begins a layer upload to repository identified by name,
|
|
|
|
// returning a handle.
|
2015-01-17 02:24:07 +00:00
|
|
|
Upload() (LayerUpload, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
|
Spool layer uploads to remote storage
To smooth initial implementation, uploads were spooled to local file storage,
validated, then pushed to remote storage. That approach was flawed in that it
present easy clustering of registry services that share a remote storage
backend. The original plan was to implement resumable hashes then implement
remote upload storage. After some thought, it was found to be better to get
remote spooling working, then optimize with resumable hashes.
Moving to this approach has tradeoffs: after storing the complete upload
remotely, the node must fetch the content and validate it before moving it to
the final location. This can double bandwidth usage to the remote backend.
Modifying the verification and upload code to store intermediate hashes should
be trivial once the layer digest format has settled.
The largest changes for users of the storage package (mostly the registry app)
are the LayerService interface and the LayerUpload interface. The LayerService
now takes qualified repository names to start and resume uploads. In corallry,
the concept of LayerUploadState has been complete removed, exposing all aspects
of that state as part of the LayerUpload object. The LayerUpload object has
been modified to work as an io.WriteSeeker and includes a StartedAt time, to
allow for upload timeout policies. Finish now only requires a digest, eliding
the requirement for a size parameter.
Resource cleanup has taken a turn for the better. Resources are cleaned up
after successful uploads and during a cancel call. Admittedly, this is probably
not completely where we want to be. It's recommend that we bolster this with a
periodic driver utility script that scans for partial uploads and deletes the
underlying data. As a small benefit, we can leave these around to better
understand how and why these uploads are failing, at the cost of some extra
disk space.
Many other changes follow from the changes above. The webapp needs to be
updated to meet the new interface requirements.
Signed-off-by: Stephen J Day <stephen.day@docker.com>
2015-01-08 22:24:02 +00:00
|
|
|
// Resume continues an in progress layer upload, returning a handle to the
|
|
|
|
// upload. The caller should seek to the latest desired upload location
|
|
|
|
// before proceeding.
|
2015-01-17 02:24:07 +00:00
|
|
|
Resume(uuid string) (LayerUpload, error)
|
2014-11-22 03:39:52 +00:00
|
|
|
}
|
2015-02-13 21:59:50 +00:00
|
|
|
|
|
|
|
// Layer provides a readable and seekable layer object. Typically,
|
|
|
|
// implementations are *not* goroutine safe.
|
|
|
|
type Layer interface {
|
|
|
|
// http.ServeContent requires an efficient implementation of
|
|
|
|
// ReadSeeker.Seek(0, os.SEEK_END).
|
|
|
|
io.ReadSeeker
|
|
|
|
io.Closer
|
|
|
|
|
2015-03-05 04:57:14 +00:00
|
|
|
// Digest returns the unique digest of the blob.
|
2015-02-13 21:59:50 +00:00
|
|
|
Digest() digest.Digest
|
|
|
|
|
2015-03-05 04:57:14 +00:00
|
|
|
// Length returns the length in bytes of the blob.
|
|
|
|
Length() int64
|
|
|
|
|
2015-02-13 21:59:50 +00:00
|
|
|
// CreatedAt returns the time this layer was created.
|
|
|
|
CreatedAt() time.Time
|
2015-03-03 16:57:52 +00:00
|
|
|
|
2015-03-13 02:31:41 +00:00
|
|
|
// Handler returns an HTTP handler which serves the layer content, whether
|
|
|
|
// by providing a redirect directly to the content, or by serving the
|
|
|
|
// content itself.
|
|
|
|
Handler(r *http.Request) (http.Handler, error)
|
2015-02-13 21:59:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// LayerUpload provides a handle for working with in-progress uploads.
|
|
|
|
// Instances can be obtained from the LayerService.Upload and
|
|
|
|
// LayerService.Resume.
|
|
|
|
type LayerUpload interface {
|
|
|
|
io.WriteSeeker
|
|
|
|
io.ReaderFrom
|
|
|
|
io.Closer
|
|
|
|
|
|
|
|
// UUID returns the identifier for this upload.
|
|
|
|
UUID() string
|
|
|
|
|
|
|
|
// StartedAt returns the time this layer upload was started.
|
|
|
|
StartedAt() time.Time
|
|
|
|
|
|
|
|
// Finish marks the upload as completed, returning a valid handle to the
|
|
|
|
// uploaded layer. The digest is validated against the contents of the
|
|
|
|
// uploaded layer.
|
|
|
|
Finish(digest digest.Digest) (Layer, error)
|
|
|
|
|
|
|
|
// Cancel the layer upload process.
|
|
|
|
Cancel() error
|
|
|
|
}
|
2015-03-04 20:32:22 +00:00
|
|
|
|
|
|
|
// SignatureService provides operations on signatures.
|
|
|
|
type SignatureService interface {
|
|
|
|
// Get retrieves all of the signature blobs for the specified digest.
|
|
|
|
Get(dgst digest.Digest) ([][]byte, error)
|
|
|
|
|
|
|
|
// Put stores the signature for the provided digest.
|
|
|
|
Put(dgst digest.Digest, signatures ...[]byte) error
|
|
|
|
}
|
2015-03-05 04:57:14 +00:00
|
|
|
|
|
|
|
// Descriptor describes targeted content. Used in conjunction with a blob
|
|
|
|
// store, a descriptor can be used to fetch, store and target any kind of
|
|
|
|
// blob. The struct also describes the wire protocol format. Fields should
|
|
|
|
// only be added but never changed.
|
|
|
|
type Descriptor struct {
|
|
|
|
// MediaType describe the type of the content. All text based formats are
|
|
|
|
// encoded as utf-8.
|
|
|
|
MediaType string `json:"mediaType,omitempty"`
|
|
|
|
|
|
|
|
// Length in bytes of content.
|
|
|
|
Length int64 `json:"length,omitempty"`
|
|
|
|
|
|
|
|
// Digest uniquely identifies the content. A byte stream can be verified
|
|
|
|
// against against this digest.
|
|
|
|
Digest digest.Digest `json:"digest,omitempty"`
|
|
|
|
|
|
|
|
// NOTE: Before adding a field here, please ensure that all
|
|
|
|
// other options have been exhausted. Much of the type relationships
|
|
|
|
// depend on the simplicity of this type.
|
|
|
|
}
|