dkirillov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Moved registry docs to registry subdirectory

Browse source
This commit is contained in:
Misty Stanley-Jones 2016-09-28 14:46:28 -07:00
parent 9f57550bb9
commit dd41410647
189 changed files with 13063 additions and 34056 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
docs/Dockerfile Normal file
View file

@ -0,0 +1,9 @@
FROM docs/base:oss
MAINTAINER Docker Docs <docs@docker.com>
ENV PROJECT=registry
# To get the git info for this repo
COPY . /src
RUN rm -rf /docs/content/$PROJECT/
COPY . /docs/content/$PROJECT/

38
docs/Makefile Normal file
View file

@ -0,0 +1,38 @@
.PHONY: all default docs docs-build docs-shell shell test
# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs)
DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR))
# to allow `make DOCSPORT=9000 docs`
DOCSPORT := 8000
# Get the IP ADDRESS
DOCKER_IP=$(shell python -c "import urlparse ; print urlparse.urlparse('$(DOCKER_HOST)').hostname or ''")
HUGO_BASE_URL=$(shell test -z "$(DOCKER_IP)" && echo localhost || echo "$(DOCKER_IP)")
HUGO_BIND_IP=0.0.0.0
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
DOCKER_DOCS_IMAGE := registry-docs$(if $(GIT_BRANCH_CLEAN),:$(GIT_BRANCH_CLEAN))
DOCKER_RUN_DOCS := docker run --rm -it $(DOCS_MOUNT) -e AWS_S3_BUCKET -e NOCACHE
# for some docs workarounds (see below in "docs-build" target)
GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null)
default: docs
docs: docs-build
$(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP)
docs-draft: docs-build
$(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP)
docs-shell: docs-build
$(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 "$(DOCKER_DOCS_IMAGE)" bash
docs-build:
docker build -t "$(DOCKER_DOCS_IMAGE)" .
test: docs-build
$(DOCKER_RUN_DOCS) "$(DOCKER_DOCS_IMAGE)"

267
docs/api/errcode/errors.go
View file

@ -1,267 +0,0 @@
package errcode
import (
"encoding/json"
"fmt"
"strings"
)
// ErrorCoder is the base interface for ErrorCode and Error allowing
// users of each to just call ErrorCode to get the real ID of each
type ErrorCoder interface {
ErrorCode() ErrorCode
}
// ErrorCode represents the error type. The errors are serialized via strings
// and the integer format may change and should *never* be exported.
type ErrorCode int
var _ error = ErrorCode(0)
// ErrorCode just returns itself
func (ec ErrorCode) ErrorCode() ErrorCode {
return ec
}
// Error returns the ID/Value
func (ec ErrorCode) Error() string {
// NOTE(stevvooe): Cannot use message here since it may have unpopulated args.
return strings.ToLower(strings.Replace(ec.String(), "_", " ", -1))
}
// Descriptor returns the descriptor for the error code.
func (ec ErrorCode) Descriptor() ErrorDescriptor {
d, ok := errorCodeToDescriptors[ec]
if !ok {
return ErrorCodeUnknown.Descriptor()
}
return d
}
// String returns the canonical identifier for this error code.
func (ec ErrorCode) String() string {
return ec.Descriptor().Value
}
// Message returned the human-readable error message for this error code.
func (ec ErrorCode) Message() string {
return ec.Descriptor().Message
}
// MarshalText encodes the receiver into UTF-8-encoded text and returns the
// result.
func (ec ErrorCode) MarshalText() (text []byte, err error) {
return []byte(ec.String()), nil
}
// UnmarshalText decodes the form generated by MarshalText.
func (ec *ErrorCode) UnmarshalText(text []byte) error {
desc, ok := idToDescriptors[string(text)]
if !ok {
desc = ErrorCodeUnknown.Descriptor()
}
*ec = desc.Code
return nil
}
// WithMessage creates a new Error struct based on the passed-in info and
// overrides the Message property.
func (ec ErrorCode) WithMessage(message string) Error {
return Error{
Code: ec,
Message: message,
}
}
// WithDetail creates a new Error struct based on the passed-in info and
// set the Detail property appropriately
func (ec ErrorCode) WithDetail(detail interface{}) Error {
return Error{
Code: ec,
Message: ec.Message(),
}.WithDetail(detail)
}
// WithArgs creates a new Error struct and sets the Args slice
func (ec ErrorCode) WithArgs(args ...interface{}) Error {
return Error{
Code: ec,
Message: ec.Message(),
}.WithArgs(args...)
}
// Error provides a wrapper around ErrorCode with extra Details provided.
type Error struct {
Code ErrorCode `json:"code"`
Message string `json:"message"`
Detail interface{} `json:"detail,omitempty"`
// TODO(duglin): See if we need an "args" property so we can do the
// variable substitution right before showing the message to the user
}
var _ error = Error{}
// ErrorCode returns the ID/Value of this Error
func (e Error) ErrorCode() ErrorCode {
return e.Code
}
// Error returns a human readable representation of the error.
func (e Error) Error() string {
return fmt.Sprintf("%s: %s", e.Code.Error(), e.Message)
}
// WithDetail will return a new Error, based on the current one, but with
// some Detail info added
func (e Error) WithDetail(detail interface{}) Error {
return Error{
Code: e.Code,
Message: e.Message,
Detail: detail,
}
}
// WithArgs uses the passed-in list of interface{} as the substitution
// variables in the Error's Message string, but returns a new Error
func (e Error) WithArgs(args ...interface{}) Error {
return Error{
Code: e.Code,
Message: fmt.Sprintf(e.Code.Message(), args...),
Detail: e.Detail,
}
}
// ErrorDescriptor provides relevant information about a given error code.
type ErrorDescriptor struct {
// Code is the error code that this descriptor describes.
Code ErrorCode
// Value provides a unique, string key, often captilized with
// underscores, to identify the error code. This value is used as the
// keyed value when serializing api errors.
Value string
// Message is a short, human readable decription of the error condition
// included in API responses.
Message string
// Description provides a complete account of the errors purpose, suitable
// for use in documentation.
Description string
// HTTPStatusCode provides the http status code that is associated with
// this error condition.
HTTPStatusCode int
}
// ParseErrorCode returns the value by the string error code.
// `ErrorCodeUnknown` will be returned if the error is not known.
func ParseErrorCode(value string) ErrorCode {
ed, ok := idToDescriptors[value]
if ok {
return ed.Code
}
return ErrorCodeUnknown
}
// Errors provides the envelope for multiple errors and a few sugar methods
// for use within the application.
type Errors []error
var _ error = Errors{}
func (errs Errors) Error() string {
switch len(errs) {
case 0:
return "<nil>"
case 1:
return errs[0].Error()
default:
msg := "errors:\n"
for _, err := range errs {
msg += err.Error() + "\n"
}
return msg
}
}
// Len returns the current number of errors.
func (errs Errors) Len() int {
return len(errs)
}
// MarshalJSON converts slice of error, ErrorCode or Error into a
// slice of Error - then serializes
func (errs Errors) MarshalJSON() ([]byte, error) {
var tmpErrs struct {
Errors []Error `json:"errors,omitempty"`
}
for _, daErr := range errs {
var err Error
switch daErr.(type) {
case ErrorCode:
err = daErr.(ErrorCode).WithDetail(nil)
case Error:
err = daErr.(Error)
default:
err = ErrorCodeUnknown.WithDetail(daErr)
}
// If the Error struct was setup and they forgot to set the
// Message field (meaning its "") then grab it from the ErrCode
msg := err.Message
if msg == "" {
msg = err.Code.Message()
}
tmpErrs.Errors = append(tmpErrs.Errors, Error{
Code: err.Code,
Message: msg,
Detail: err.Detail,
})
}
return json.Marshal(tmpErrs)
}
// UnmarshalJSON deserializes []Error and then converts it into slice of
// Error or ErrorCode
func (errs *Errors) UnmarshalJSON(data []byte) error {
var tmpErrs struct {
Errors []Error
}
if err := json.Unmarshal(data, &tmpErrs); err != nil {
return err
}
var newErrs Errors
for _, daErr := range tmpErrs.Errors {
// If Message is empty or exactly matches the Code's message string
// then just use the Code, no need for a full Error struct
if daErr.Detail == nil && (daErr.Message == "" || daErr.Message == daErr.Code.Message()) {
// Error's w/o details get converted to ErrorCode
newErrs = append(newErrs, daErr.Code)
} else {
// Error's w/ details are untouched
newErrs = append(newErrs, Error{
Code: daErr.Code,
Message: daErr.Message,
Detail: daErr.Detail,
})
}
}
*errs = newErrs
return nil
}

185
docs/api/errcode/errors_test.go
View file

@ -1,185 +0,0 @@
package errcode
import (
"encoding/json"
"net/http"
"reflect"
"strings"
"testing"
)
// TestErrorsManagement does a quick check of the Errors type to ensure that
// members are properly pushed and marshaled.
var ErrorCodeTest1 = Register("test.errors", ErrorDescriptor{
Value: "TEST1",
Message: "test error 1",
Description: `Just a test message #1.`,
HTTPStatusCode: http.StatusInternalServerError,
})
var ErrorCodeTest2 = Register("test.errors", ErrorDescriptor{
Value: "TEST2",
Message: "test error 2",
Description: `Just a test message #2.`,
HTTPStatusCode: http.StatusNotFound,
})
var ErrorCodeTest3 = Register("test.errors", ErrorDescriptor{
Value: "TEST3",
Message: "Sorry %q isn't valid",
Description: `Just a test message #3.`,
HTTPStatusCode: http.StatusNotFound,
})
// TestErrorCodes ensures that error code format, mappings and
// marshaling/unmarshaling. round trips are stable.
func TestErrorCodes(t *testing.T) {
if len(errorCodeToDescriptors) == 0 {
t.Fatal("errors aren't loaded!")
}
for ec, desc := range errorCodeToDescriptors {
if ec != desc.Code {
t.Fatalf("error code in descriptor isn't correct, %q != %q", ec, desc.Code)
}
if idToDescriptors[desc.Value].Code != ec {
t.Fatalf("error code in idToDesc isn't correct, %q != %q", idToDescriptors[desc.Value].Code, ec)
}
if ec.Message() != desc.Message {
t.Fatalf("ec.Message doesn't mtach desc.Message: %q != %q", ec.Message(), desc.Message)
}
// Test (de)serializing the ErrorCode
p, err := json.Marshal(ec)
if err != nil {
t.Fatalf("couldn't marshal ec %v: %v", ec, err)
}
if len(p) <= 0 {
t.Fatalf("expected content in marshaled before for error code %v", ec)
}
// First, unmarshal to interface and ensure we have a string.
var ecUnspecified interface{}
if err := json.Unmarshal(p, &ecUnspecified); err != nil {
t.Fatalf("error unmarshaling error code %v: %v", ec, err)
}
if _, ok := ecUnspecified.(string); !ok {
t.Fatalf("expected a string for error code %v on unmarshal got a %T", ec, ecUnspecified)
}
// Now, unmarshal with the error code type and ensure they are equal
var ecUnmarshaled ErrorCode
if err := json.Unmarshal(p, &ecUnmarshaled); err != nil {
t.Fatalf("error unmarshaling error code %v: %v", ec, err)
}
if ecUnmarshaled != ec {
t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, ec)
}
expectedErrorString := strings.ToLower(strings.Replace(ec.Descriptor().Value, "_", " ", -1))
if ec.Error() != expectedErrorString {
t.Fatalf("unexpected return from %v.Error(): %q != %q", ec, ec.Error(), expectedErrorString)
}
}
}
func TestErrorsManagement(t *testing.T) {
var errs Errors
errs = append(errs, ErrorCodeTest1)
errs = append(errs, ErrorCodeTest2.WithDetail(
map[string]interface{}{"digest": "sometestblobsumdoesntmatter"}))
errs = append(errs, ErrorCodeTest3.WithArgs("BOOGIE"))
errs = append(errs, ErrorCodeTest3.WithArgs("BOOGIE").WithDetail("data"))
p, err := json.Marshal(errs)
if err != nil {
t.Fatalf("error marashaling errors: %v", err)
}
expectedJSON := `{"errors":[` +
`{"code":"TEST1","message":"test error 1"},` +
`{"code":"TEST2","message":"test error 2","detail":{"digest":"sometestblobsumdoesntmatter"}},` +
`{"code":"TEST3","message":"Sorry \"BOOGIE\" isn't valid"},` +
`{"code":"TEST3","message":"Sorry \"BOOGIE\" isn't valid","detail":"data"}` +
`]}`
if string(p) != expectedJSON {
t.Fatalf("unexpected json:\ngot:\n%q\n\nexpected:\n%q", string(p), expectedJSON)
}
// Now test the reverse
var unmarshaled Errors
if err := json.Unmarshal(p, &unmarshaled); err != nil {
t.Fatalf("unexpected error unmarshaling error envelope: %v", err)
}
if !reflect.DeepEqual(unmarshaled, errs) {
t.Fatalf("errors not equal after round trip:\nunmarshaled:\n%#v\n\nerrs:\n%#v", unmarshaled, errs)
}
// Test the arg substitution stuff
e1 := unmarshaled[3].(Error)
exp1 := `Sorry "BOOGIE" isn't valid`
if e1.Message != exp1 {
t.Fatalf("Wrong msg, got:\n%q\n\nexpected:\n%q", e1.Message, exp1)
}
exp1 = "test3: " + exp1
if e1.Error() != exp1 {
t.Fatalf("Error() didn't return the right string, got:%s\nexpected:%s", e1.Error(), exp1)
}
// Test again with a single value this time
errs = Errors{ErrorCodeUnknown}
expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}"
p, err = json.Marshal(errs)
if err != nil {
t.Fatalf("error marashaling errors: %v", err)
}
if string(p) != expectedJSON {
t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON)
}
// Now test the reverse
unmarshaled = nil
if err := json.Unmarshal(p, &unmarshaled); err != nil {
t.Fatalf("unexpected error unmarshaling error envelope: %v", err)
}
if !reflect.DeepEqual(unmarshaled, errs) {
t.Fatalf("errors not equal after round trip:\nunmarshaled:\n%#v\n\nerrs:\n%#v", unmarshaled, errs)
}
// Verify that calling WithArgs() more than once does the right thing.
// Meaning creates a new Error and uses the ErrorCode Message
e1 = ErrorCodeTest3.WithArgs("test1")
e2 := e1.WithArgs("test2")
if &e1 == &e2 {
t.Fatalf("args: e2 and e1 should not be the same, but they are")
}
if e2.Message != `Sorry "test2" isn't valid` {
t.Fatalf("e2 had wrong message: %q", e2.Message)
}
// Verify that calling WithDetail() more than once does the right thing.
// Meaning creates a new Error and overwrites the old detail field
e1 = ErrorCodeTest3.WithDetail("stuff1")
e2 = e1.WithDetail("stuff2")
if &e1 == &e2 {
t.Fatalf("detail: e2 and e1 should not be the same, but they are")
}
if e2.Detail != `stuff2` {
t.Fatalf("e2 had wrong detail: %q", e2.Detail)
}
}

44
docs/api/errcode/handler.go
View file

@ -1,44 +0,0 @@
package errcode
import (
"encoding/json"
"net/http"
)
// ServeJSON attempts to serve the errcode in a JSON envelope. It marshals err
// and sets the content-type header to 'application/json'. It will handle
// ErrorCoder and Errors, and if necessary will create an envelope.
func ServeJSON(w http.ResponseWriter, err error) error {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
var sc int
switch errs := err.(type) {
case Errors:
if len(errs) < 1 {
break
}
if err, ok := errs[0].(ErrorCoder); ok {
sc = err.ErrorCode().Descriptor().HTTPStatusCode
}
case ErrorCoder:
sc = errs.ErrorCode().Descriptor().HTTPStatusCode
err = Errors{err} // create an envelope.
default:
// We just have an unhandled error type, so just place in an envelope
// and move along.
err = Errors{err}
}
if sc == 0 {
sc = http.StatusInternalServerError
}
w.WriteHeader(sc)
if err := json.NewEncoder(w).Encode(err); err != nil {
return err
}
return nil
}

138
docs/api/errcode/register.go
View file

@ -1,138 +0,0 @@
package errcode
import (
"fmt"
"net/http"
"sort"
"sync"
)
var (
errorCodeToDescriptors = map[ErrorCode]ErrorDescriptor{}
idToDescriptors = map[string]ErrorDescriptor{}
groupToDescriptors = map[string][]ErrorDescriptor{}
)
var (
// ErrorCodeUnknown is a generic error that can be used as a last
// resort if there is no situation-specific error message that can be used
ErrorCodeUnknown = Register("errcode", ErrorDescriptor{
Value: "UNKNOWN",
Message: "unknown error",
Description: `Generic error returned when the error does not have an
API classification.`,
HTTPStatusCode: http.StatusInternalServerError,
})
// ErrorCodeUnsupported is returned when an operation is not supported.
ErrorCodeUnsupported = Register("errcode", ErrorDescriptor{
Value: "UNSUPPORTED",
Message: "The operation is unsupported.",
Description: `The operation was unsupported due to a missing
implementation or invalid set of parameters.`,
HTTPStatusCode: http.StatusMethodNotAllowed,
})
// ErrorCodeUnauthorized is returned if a request requires
// authentication.
ErrorCodeUnauthorized = Register("errcode", ErrorDescriptor{
Value: "UNAUTHORIZED",
Message: "authentication required",
Description: `The access controller was unable to authenticate
the client. Often this will be accompanied by a
Www-Authenticate HTTP response header indicating how to
authenticate.`,
HTTPStatusCode: http.StatusUnauthorized,
})
// ErrorCodeDenied is returned if a client does not have sufficient
// permission to perform an action.
ErrorCodeDenied = Register("errcode", ErrorDescriptor{
Value: "DENIED",
Message: "requested access to the resource is denied",
Description: `The access controller denied access for the
operation on a resource.`,
HTTPStatusCode: http.StatusForbidden,
})
// ErrorCodeUnavailable provides a common error to report unavailability
// of a service or endpoint.
ErrorCodeUnavailable = Register("errcode", ErrorDescriptor{
Value: "UNAVAILABLE",
Message: "service unavailable",
Description: "Returned when a service is not available",
HTTPStatusCode: http.StatusServiceUnavailable,
})
// ErrorCodeTooManyRequests is returned if a client attempts too many
// times to contact a service endpoint.
ErrorCodeTooManyRequests = Register("errcode", ErrorDescriptor{
Value: "TOOMANYREQUESTS",
Message: "too many requests",
Description: `Returned when a client attempts to contact a
service too many times`,
HTTPStatusCode: http.StatusTooManyRequests,
})
)
var nextCode = 1000
var registerLock sync.Mutex
// Register will make the passed-in error known to the environment and
// return a new ErrorCode
func Register(group string, descriptor ErrorDescriptor) ErrorCode {
registerLock.Lock()
defer registerLock.Unlock()
descriptor.Code = ErrorCode(nextCode)
if _, ok := idToDescriptors[descriptor.Value]; ok {
panic(fmt.Sprintf("ErrorValue %q is already registered", descriptor.Value))
}
if _, ok := errorCodeToDescriptors[descriptor.Code]; ok {
panic(fmt.Sprintf("ErrorCode %v is already registered", descriptor.Code))
}
groupToDescriptors[group] = append(groupToDescriptors[group], descriptor)
errorCodeToDescriptors[descriptor.Code] = descriptor
idToDescriptors[descriptor.Value] = descriptor
nextCode++
return descriptor.Code
}
type byValue []ErrorDescriptor
func (a byValue) Len() int { return len(a) }
func (a byValue) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byValue) Less(i, j int) bool { return a[i].Value < a[j].Value }
// GetGroupNames returns the list of Error group names that are registered
func GetGroupNames() []string {
keys := []string{}
for k := range groupToDescriptors {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// GetErrorCodeGroup returns the named group of error descriptors
func GetErrorCodeGroup(name string) []ErrorDescriptor {
desc := groupToDescriptors[name]
sort.Sort(byValue(desc))
return desc
}
// GetErrorAllDescriptors returns a slice of all ErrorDescriptors that are
// registered, irrespective of what group they're in
func GetErrorAllDescriptors() []ErrorDescriptor {
result := []ErrorDescriptor{}
for _, group := range GetGroupNames() {
result = append(result, GetErrorCodeGroup(group)...)
}
sort.Sort(byValue(result))
return result
}

1596
docs/api/v2/descriptors.go
View file

File diff suppressed because it is too large Load diff

9
docs/api/v2/doc.go
View file

@ -1,9 +0,0 @@
// Package v2 describes routes, urls and the error codes used in the Docker
// Registry JSON HTTP API V2. In addition to declarations, descriptors are
// provided for routes and error codes that can be used for implementation and
// automatically generating documentation.
//
// Definitions here are considered to be locked down for the V2 registry api.
// Any changes must be considered carefully and should not proceed without a
// change proposal in docker core.
package v2

136
docs/api/v2/errors.go
View file

@ -1,136 +0,0 @@
package v2
import (
"net/http"
"github.com/docker/distribution/registry/api/errcode"
)
const errGroup = "registry.api.v2"
var (
// ErrorCodeDigestInvalid is returned when uploading a blob if the
// provided digest does not match the blob contents.
ErrorCodeDigestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "DIGEST_INVALID",
Message: "provided digest did not match uploaded content",
Description: `When a blob is uploaded, the registry will check that
the content matches the digest provided by the client. The error may
include a detail structure with the key "digest", including the
invalid digest string. This error may also be returned when a manifest
includes an invalid layer digest.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeSizeInvalid is returned when uploading a blob if the provided
ErrorCodeSizeInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "SIZE_INVALID",
Message: "provided length did not match content length",
Description: `When a layer is uploaded, the provided size will be
checked against the uploaded content. If they do not match, this error
will be returned.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeNameInvalid is returned when the name in the manifest does not
// match the provided name.
ErrorCodeNameInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "NAME_INVALID",
Message: "invalid repository name",
Description: `Invalid repository name encountered either during
manifest validation or any API operation.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeTagInvalid is returned when the tag in the manifest does not
// match the provided tag.
ErrorCodeTagInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "TAG_INVALID",
Message: "manifest tag did not match URI",
Description: `During a manifest upload, if the tag in the manifest
does not match the uri tag, this error will be returned.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeNameUnknown when the repository name is not known.
ErrorCodeNameUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "NAME_UNKNOWN",
Message: "repository name not known to registry",
Description: `This is returned if the name used during an operation is
unknown to the registry.`,
HTTPStatusCode: http.StatusNotFound,
})
// ErrorCodeManifestUnknown returned when image manifest is unknown.
ErrorCodeManifestUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "MANIFEST_UNKNOWN",
Message: "manifest unknown",
Description: `This error is returned when the manifest, identified by
name and tag is unknown to the repository.`,
HTTPStatusCode: http.StatusNotFound,
})
// ErrorCodeManifestInvalid returned when an image manifest is invalid,
// typically during a PUT operation. This error encompasses all errors
// encountered during manifest validation that aren't signature errors.
ErrorCodeManifestInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "MANIFEST_INVALID",
Message: "manifest invalid",
Description: `During upload, manifests undergo several checks ensuring
validity. If those checks fail, this error may be returned, unless a
more specific error is included. The detail will contain information
the failed validation.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeManifestUnverified is returned when the manifest fails
// signature verification.
ErrorCodeManifestUnverified = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "MANIFEST_UNVERIFIED",
Message: "manifest failed signature verification",
Description: `During manifest upload, if the manifest fails signature
verification, this error will be returned.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeManifestBlobUnknown is returned when a manifest blob is
// unknown to the registry.
ErrorCodeManifestBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "MANIFEST_BLOB_UNKNOWN",
Message: "blob unknown to registry",
Description: `This error may be returned when a manifest blob is
unknown to the registry.`,
HTTPStatusCode: http.StatusBadRequest,
})
// ErrorCodeBlobUnknown is returned when a blob is unknown to the
// registry. This can happen when the manifest references a nonexistent
// layer or the result is not found by a blob fetch.
ErrorCodeBlobUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "BLOB_UNKNOWN",
Message: "blob unknown to registry",
Description: `This error may be returned when a blob is unknown to the
registry in a specified repository. This can be returned with a
standard get or if a manifest references an unknown layer during
upload.`,
HTTPStatusCode: http.StatusNotFound,
})
// ErrorCodeBlobUploadUnknown is returned when an upload is unknown.
ErrorCodeBlobUploadUnknown = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "BLOB_UPLOAD_UNKNOWN",
Message: "blob upload unknown to registry",
Description: `If a blob upload has been cancelled or was never
started, this error code may be returned.`,
HTTPStatusCode: http.StatusNotFound,
})
// ErrorCodeBlobUploadInvalid is returned when an upload is invalid.
ErrorCodeBlobUploadInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{
Value: "BLOB_UPLOAD_INVALID",
Message: "blob upload invalid",
Description: `The blob upload encountered an error and can no
longer proceed.`,
HTTPStatusCode: http.StatusNotFound,
})
)

49
docs/api/v2/routes.go
View file

@ -1,49 +0,0 @@
package v2
import "github.com/gorilla/mux"
// The following are definitions of the name under which all V2 routes are
// registered. These symbols can be used to look up a route based on the name.
const (
RouteNameBase = "base"
RouteNameManifest = "manifest"
RouteNameTags = "tags"
RouteNameBlob = "blob"
RouteNameBlobUpload = "blob-upload"
RouteNameBlobUploadChunk = "blob-upload-chunk"
RouteNameCatalog = "catalog"
)
var allEndpoints = []string{
RouteNameManifest,
RouteNameCatalog,
RouteNameTags,
RouteNameBlob,
RouteNameBlobUpload,
RouteNameBlobUploadChunk,
}
// Router builds a gorilla router with named routes for the various API
// methods. This can be used directly by both server implementations and
// clients.
func Router() *mux.Router {
return RouterWithPrefix("")
}
// RouterWithPrefix builds a gorilla router with a configured prefix
// on all routes.
func RouterWithPrefix(prefix string) *mux.Router {
rootRouter := mux.NewRouter()
router := rootRouter
if prefix != "" {
router = router.PathPrefix(prefix).Subrouter()
}
router.StrictSlash(true)
for _, descriptor := range routeDescriptors {
router.Path(descriptor.Path).Name(descriptor.Name)
}
return rootRouter
}

355
docs/api/v2/routes_test.go
View file

@ -1,355 +0,0 @@
package v2
import (
"encoding/json"
"fmt"
"math/rand"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
"github.com/gorilla/mux"
)
type routeTestCase struct {
RequestURI string
ExpectedURI string
Vars map[string]string
RouteName string
StatusCode int
}
// TestRouter registers a test handler with all the routes and ensures that
// each route returns the expected path variables. Not method verification is
// present. This not meant to be exhaustive but as check to ensure that the
// expected variables are extracted.
//
// This may go away as the application structure comes together.
func TestRouter(t *testing.T) {
testCases := []routeTestCase{
{
RouteName: RouteNameBase,
RequestURI: "/v2/",
Vars: map[string]string{},
},
{
RouteName: RouteNameManifest,
RequestURI: "/v2/foo/manifests/bar",
Vars: map[string]string{
"name": "foo",
"reference": "bar",
},
},
{
RouteName: RouteNameManifest,
RequestURI: "/v2/foo/bar/manifests/tag",
Vars: map[string]string{
"name": "foo/bar",
"reference": "tag",
},
},
{
RouteName: RouteNameManifest,
RequestURI: "/v2/foo/bar/manifests/sha256:abcdef01234567890",
Vars: map[string]string{
"name": "foo/bar",
"reference": "sha256:abcdef01234567890",
},
},
{
RouteName: RouteNameTags,
RequestURI: "/v2/foo/bar/tags/list",
Vars: map[string]string{
"name": "foo/bar",
},
},
{
RouteName: RouteNameTags,
RequestURI: "/v2/docker.com/foo/tags/list",
Vars: map[string]string{
"name": "docker.com/foo",
},
},
{
RouteName: RouteNameTags,
RequestURI: "/v2/docker.com/foo/bar/tags/list",
Vars: map[string]string{
"name": "docker.com/foo/bar",
},
},
{
RouteName: RouteNameTags,
RequestURI: "/v2/docker.com/foo/bar/baz/tags/list",
Vars: map[string]string{
"name": "docker.com/foo/bar/baz",
},
},
{
RouteName: RouteNameBlob,
RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234",
Vars: map[string]string{
"name": "foo/bar",
"digest": "sha256:abcdef0919234",
},
},
{
RouteName: RouteNameBlobUpload,
RequestURI: "/v2/foo/bar/blobs/uploads/",
Vars: map[string]string{
"name": "foo/bar",
},
},
{
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/bar/blobs/uploads/uuid",
Vars: map[string]string{
"name": "foo/bar",
"uuid": "uuid",
},
},
{
// support uuid proper
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286",
Vars: map[string]string{
"name": "foo/bar",
"uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286",
},
},
{
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==",
Vars: map[string]string{
"name": "foo/bar",
"uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==",
},
},
{
// supports urlsafe base64
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA_-==",
Vars: map[string]string{
"name": "foo/bar",
"uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA_-==",
},
},
{
// does not match
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/bar/blobs/uploads/totalandcompletejunk++$$-==",
StatusCode: http.StatusNotFound,
},
{
// Check ambiguity: ensure we can distinguish between tags for
// "foo/bar/image/image" and image for "foo/bar/image" with tag
// "tags"
RouteName: RouteNameManifest,
RequestURI: "/v2/foo/bar/manifests/manifests/tags",
Vars: map[string]string{
"name": "foo/bar/manifests",
"reference": "tags",
},
},
{
// This case presents an ambiguity between foo/bar with tag="tags"
// and list tags for "foo/bar/manifest"
RouteName: RouteNameTags,
RequestURI: "/v2/foo/bar/manifests/tags/list",
Vars: map[string]string{
"name": "foo/bar/manifests",
},
},
{
RouteName: RouteNameManifest,
RequestURI: "/v2/locahost:8080/foo/bar/baz/manifests/tag",
Vars: map[string]string{
"name": "locahost:8080/foo/bar/baz",
"reference": "tag",
},
},
}
checkTestRouter(t, testCases, "", true)
checkTestRouter(t, testCases, "/prefix/", true)
}
func TestRouterWithPathTraversals(t *testing.T) {
testCases := []routeTestCase{
{
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286",
ExpectedURI: "/blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286",
StatusCode: http.StatusNotFound,
},
{
// Testing for path traversal attack handling
RouteName: RouteNameTags,
RequestURI: "/v2/foo/../bar/baz/tags/list",
ExpectedURI: "/v2/bar/baz/tags/list",
Vars: map[string]string{
"name": "bar/baz",
},
},
}
checkTestRouter(t, testCases, "", false)
}
func TestRouterWithBadCharacters(t *testing.T) {
if testing.Short() {
testCases := []routeTestCase{
{
RouteName: RouteNameBlobUploadChunk,
RequestURI: "/v2/foo/blob/uploads/不95306FA-FAD3-4E36-8D41-CF1C93EF8286",
StatusCode: http.StatusNotFound,
},
{
// Testing for path traversal attack handling
RouteName: RouteNameTags,
RequestURI: "/v2/foo/不bar/tags/list",
StatusCode: http.StatusNotFound,
},
}
checkTestRouter(t, testCases, "", true)
} else {
// in the long version we're going to fuzz the router
// with random UTF8 characters not in the 128 bit ASCII range.
// These are not valid characters for the router and we expect
// 404s on every test.
rand.Seed(time.Now().UTC().UnixNano())
testCases := make([]routeTestCase, 1000)
for idx := range testCases {
testCases[idx] = routeTestCase{
RouteName: RouteNameTags,
RequestURI: fmt.Sprintf("/v2/%v/%v/tags/list", randomString(10), randomString(10)),
StatusCode: http.StatusNotFound,
}
}
checkTestRouter(t, testCases, "", true)
}
}
func checkTestRouter(t *testing.T, testCases []routeTestCase, prefix string, deeplyEqual bool) {
router := RouterWithPrefix(prefix)
testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
testCase := routeTestCase{
RequestURI: r.RequestURI,
Vars: mux.Vars(r),
RouteName: mux.CurrentRoute(r).GetName(),
}
enc := json.NewEncoder(w)
if err := enc.Encode(testCase); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
})
// Startup test server
server := httptest.NewServer(router)
for _, testcase := range testCases {
testcase.RequestURI = strings.TrimSuffix(prefix, "/") + testcase.RequestURI
// Register the endpoint
route := router.GetRoute(testcase.RouteName)
if route == nil {
t.Fatalf("route for name %q not found", testcase.RouteName)
}
route.Handler(testHandler)
u := server.URL + testcase.RequestURI
resp, err := http.Get(u)
if err != nil {
t.Fatalf("error issuing get request: %v", err)
}
if testcase.StatusCode == 0 {
// Override default, zero-value
testcase.StatusCode = http.StatusOK
}
if testcase.ExpectedURI == "" {
// Override default, zero-value
testcase.ExpectedURI = testcase.RequestURI
}
if resp.StatusCode != testcase.StatusCode {
t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode)
}
if testcase.StatusCode != http.StatusOK {
resp.Body.Close()
// We don't care about json response.
continue
}
dec := json.NewDecoder(resp.Body)
var actualRouteInfo routeTestCase
if err := dec.Decode(&actualRouteInfo); err != nil {
t.Fatalf("error reading json response: %v", err)
}
// Needs to be set out of band
actualRouteInfo.StatusCode = resp.StatusCode
if actualRouteInfo.RequestURI != testcase.ExpectedURI {
t.Fatalf("URI %v incorrectly parsed, expected %v", actualRouteInfo.RequestURI, testcase.ExpectedURI)
}
if actualRouteInfo.RouteName != testcase.RouteName {
t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName)
}
// when testing deep equality, the actualRouteInfo has an empty ExpectedURI, we don't want
// that to make the comparison fail. We're otherwise done with the testcase so empty the
// testcase.ExpectedURI
testcase.ExpectedURI = ""
if deeplyEqual && !reflect.DeepEqual(actualRouteInfo, testcase) {
t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase)
}
resp.Body.Close()
}
}
// -------------- START LICENSED CODE --------------
// The following code is derivative of https://github.com/google/gofuzz
// gofuzz is licensed under the Apache License, Version 2.0, January 2004,
// a copy of which can be found in the LICENSE file at the root of this
// repository.
// These functions allow us to generate strings containing only multibyte
// characters that are invalid in our URLs. They are used above for fuzzing
// to ensure we always get 404s on these invalid strings
type charRange struct {
first, last rune
}
// choose returns a random unicode character from the given range, using the
// given randomness source.
func (r *charRange) choose() rune {
count := int64(r.last - r.first)
return r.first + rune(rand.Int63n(count))
}
var unicodeRanges = []charRange{
{'\u00a0', '\u02af'}, // Multi-byte encoded characters
{'\u4e00', '\u9fff'}, // Common CJK (even longer encodings)
}
func randomString(length int) string {
runes := make([]rune, length)
for i := range runes {
runes[i] = unicodeRanges[rand.Intn(len(unicodeRanges))].choose()
}
return string(runes)
}
// -------------- END LICENSED CODE --------------

251
docs/api/v2/urls.go
View file

@ -1,251 +0,0 @@
package v2
import (
"net/http"
"net/url"
"strings"
"github.com/docker/distribution/reference"
"github.com/gorilla/mux"
)
// URLBuilder creates registry API urls from a single base endpoint. It can be
// used to create urls for use in a registry client or server.
//
// All urls will be created from the given base, including the api version.
// For example, if a root of "/foo/" is provided, urls generated will be fall
// under "/foo/v2/...". Most application will only provide a schema, host and
// port, such as "https://localhost:5000/".
type URLBuilder struct {
root *url.URL // url root (ie http://localhost/)
router *mux.Router
relative bool
}
// NewURLBuilder creates a URLBuilder with provided root url object.
func NewURLBuilder(root *url.URL, relative bool) *URLBuilder {
return &URLBuilder{
root: root,
router: Router(),
relative: relative,
}
}
// NewURLBuilderFromString workes identically to NewURLBuilder except it takes
// a string argument for the root, returning an error if it is not a valid
// url.
func NewURLBuilderFromString(root string, relative bool) (*URLBuilder, error) {
u, err := url.Parse(root)
if err != nil {
return nil, err
}
return NewURLBuilder(u, relative), nil
}
// NewURLBuilderFromRequest uses information from an *http.Request to
// construct the root url.
func NewURLBuilderFromRequest(r *http.Request, relative bool) *URLBuilder {
var scheme string
forwardedProto := r.Header.Get("X-Forwarded-Proto")
switch {
case len(forwardedProto) > 0:
scheme = forwardedProto
case r.TLS != nil:
scheme = "https"
case len(r.URL.Scheme) > 0:
scheme = r.URL.Scheme
default:
scheme = "http"
}
host := r.Host
forwardedHost := r.Header.Get("X-Forwarded-Host")
if len(forwardedHost) > 0 {
// According to the Apache mod_proxy docs, X-Forwarded-Host can be a
// comma-separated list of hosts, to which each proxy appends the
// requested host. We want to grab the first from this comma-separated
// list.
hosts := strings.SplitN(forwardedHost, ",", 2)
host = strings.TrimSpace(hosts[0])
}
basePath := routeDescriptorsMap[RouteNameBase].Path
requestPath := r.URL.Path
index := strings.Index(requestPath, basePath)
u := &url.URL{
Scheme: scheme,
Host: host,
}
if index > 0 {
// N.B. index+1 is important because we want to include the trailing /
u.Path = requestPath[0 : index+1]
}
return NewURLBuilder(u, relative)
}
// BuildBaseURL constructs a base url for the API, typically just "/v2/".
func (ub *URLBuilder) BuildBaseURL() (string, error) {
route := ub.cloneRoute(RouteNameBase)
baseURL, err := route.URL()
if err != nil {
return "", err
}
return baseURL.String(), nil
}
// BuildCatalogURL constructs a url get a catalog of repositories
func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) {
route := ub.cloneRoute(RouteNameCatalog)
catalogURL, err := route.URL()
if err != nil {
return "", err
}
return appendValuesURL(catalogURL, values...).String(), nil
}
// BuildTagsURL constructs a url to list the tags in the named repository.
func (ub *URLBuilder) BuildTagsURL(name reference.Named) (string, error) {
route := ub.cloneRoute(RouteNameTags)
tagsURL, err := route.URL("name", name.Name())
if err != nil {
return "", err
}
return tagsURL.String(), nil
}
// BuildManifestURL constructs a url for the manifest identified by name and
// reference. The argument reference may be either a tag or digest.
func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) {
route := ub.cloneRoute(RouteNameManifest)
tagOrDigest := ""
switch v := ref.(type) {
case reference.Tagged:
tagOrDigest = v.Tag()
case reference.Digested:
tagOrDigest = v.Digest().String()
}
manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest)
if err != nil {
return "", err
}
return manifestURL.String(), nil
}
// BuildBlobURL constructs the url for the blob identified by name and dgst.
func (ub *URLBuilder) BuildBlobURL(ref reference.Canonical) (string, error) {
route := ub.cloneRoute(RouteNameBlob)
layerURL, err := route.URL("name", ref.Name(), "digest", ref.Digest().String())
if err != nil {
return "", err
}
return layerURL.String(), nil
}
// BuildBlobUploadURL constructs a url to begin a blob upload in the
// repository identified by name.
func (ub *URLBuilder) BuildBlobUploadURL(name reference.Named, values ...url.Values) (string, error) {
route := ub.cloneRoute(RouteNameBlobUpload)
uploadURL, err := route.URL("name", name.Name())
if err != nil {
return "", err
}
return appendValuesURL(uploadURL, values...).String(), nil
}
// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid,
// including any url values. This should generally not be used by clients, as
// this url is provided by server implementations during the blob upload
// process.
func (ub *URLBuilder) BuildBlobUploadChunkURL(name reference.Named, uuid string, values ...url.Values) (string, error) {
route := ub.cloneRoute(RouteNameBlobUploadChunk)
uploadURL, err := route.URL("name", name.Name(), "uuid", uuid)
if err != nil {
return "", err
}
return appendValuesURL(uploadURL, values...).String(), nil
}
// clondedRoute returns a clone of the named route from the router. Routes
// must be cloned to avoid modifying them during url generation.
func (ub *URLBuilder) cloneRoute(name string) clonedRoute {
route := new(mux.Route)
root := new(url.URL)
*route = *ub.router.GetRoute(name) // clone the route
*root = *ub.root
return clonedRoute{Route: route, root: root, relative: ub.relative}
}
type clonedRoute struct {
*mux.Route
root *url.URL
relative bool
}
func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) {
routeURL, err := cr.Route.URL(pairs...)
if err != nil {
return nil, err
}
if cr.relative {
return routeURL, nil
}
if routeURL.Scheme == "" && routeURL.User == nil && routeURL.Host == "" {
routeURL.Path = routeURL.Path[1:]
}
url := cr.root.ResolveReference(routeURL)
url.Scheme = cr.root.Scheme
return url, nil
}
// appendValuesURL appends the parameters to the url.
func appendValuesURL(u *url.URL, values ...url.Values) *url.URL {
merged := u.Query()
for _, v := range values {
for k, vv := range v {
merged[k] = append(merged[k], vv...)
}
}
u.RawQuery = merged.Encode()
return u
}
// appendValues appends the parameters to the url. Panics if the string is not
// a url.
func appendValues(u string, values ...url.Values) string {
up, err := url.Parse(u)
if err != nil {
panic(err) // should never happen
}
return appendValuesURL(up, values...).String()
}

334
docs/api/v2/urls_test.go
View file

@ -1,334 +0,0 @@
package v2
import (
"net/http"
"net/url"
"testing"
"github.com/docker/distribution/reference"
)
type urlBuilderTestCase struct {
description string
expectedPath string
build func() (string, error)
}
func makeURLBuilderTestCases(urlBuilder *URLBuilder) []urlBuilderTestCase {
fooBarRef, _ := reference.ParseNamed("foo/bar")
return []urlBuilderTestCase{
{
description: "test base url",
expectedPath: "/v2/",
build: urlBuilder.BuildBaseURL,
},
{
description: "test tags url",
expectedPath: "/v2/foo/bar/tags/list",
build: func() (string, error) {
return urlBuilder.BuildTagsURL(fooBarRef)
},
},
{
description: "test manifest url",
expectedPath: "/v2/foo/bar/manifests/tag",
build: func() (string, error) {
ref, _ := reference.WithTag(fooBarRef, "tag")
return urlBuilder.BuildManifestURL(ref)
},
},
{
description: "build blob url",
expectedPath: "/v2/foo/bar/blobs/sha256:3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5",
build: func() (string, error) {
ref, _ := reference.WithDigest(fooBarRef, "sha256:3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5")
return urlBuilder.BuildBlobURL(ref)
},
},
{
description: "build blob upload url",
expectedPath: "/v2/foo/bar/blobs/uploads/",
build: func() (string, error) {
return urlBuilder.BuildBlobUploadURL(fooBarRef)
},
},
{
description: "build blob upload url with digest and size",
expectedPath: "/v2/foo/bar/blobs/uploads/?digest=sha256%3A3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5&size=10000",
build: func() (string, error) {
return urlBuilder.BuildBlobUploadURL(fooBarRef, url.Values{
"size": []string{"10000"},
"digest": []string{"sha256:3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5"},
})
},
},
{
description: "build blob upload chunk url",
expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part",
build: func() (string, error) {
return urlBuilder.BuildBlobUploadChunkURL(fooBarRef, "uuid-part")
},
},
{
description: "build blob upload chunk url with digest and size",
expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part?digest=sha256%3A3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5&size=10000",
build: func() (string, error) {
return urlBuilder.BuildBlobUploadChunkURL(fooBarRef, "uuid-part", url.Values{
"size": []string{"10000"},
"digest": []string{"sha256:3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5"},
})
},
},
}
}
// TestURLBuilder tests the various url building functions, ensuring they are
// returning the expected values.
func TestURLBuilder(t *testing.T) {
roots := []string{
"http://example.com",
"https://example.com",
"http://localhost:5000",
"https://localhost:5443",
}
doTest := func(relative bool) {
for _, root := range roots {
urlBuilder, err := NewURLBuilderFromString(root, relative)
if err != nil {
t.Fatalf("unexpected error creating urlbuilder: %v", err)
}
for _, testCase := range makeURLBuilderTestCases(urlBuilder) {
url, err := testCase.build()
if err != nil {
t.Fatalf("%s: error building url: %v", testCase.description, err)
}
expectedURL := testCase.expectedPath
if !relative {
expectedURL = root + expectedURL
}
if url != expectedURL {
t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL)
}
}
}
}
doTest(true)
doTest(false)
}
func TestURLBuilderWithPrefix(t *testing.T) {
roots := []string{
"http://example.com/prefix/",
"https://example.com/prefix/",
"http://localhost:5000/prefix/",
"https://localhost:5443/prefix/",
}
doTest := func(relative bool) {
for _, root := range roots {
urlBuilder, err := NewURLBuilderFromString(root, relative)
if err != nil {
t.Fatalf("unexpected error creating urlbuilder: %v", err)
}
for _, testCase := range makeURLBuilderTestCases(urlBuilder) {
url, err := testCase.build()
if err != nil {
t.Fatalf("%s: error building url: %v", testCase.description, err)
}
expectedURL := testCase.expectedPath
if !relative {
expectedURL = root[0:len(root)-1] + expectedURL
}
if url != expectedURL {
t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL)
}
}
}
}
doTest(true)
doTest(false)
}
type builderFromRequestTestCase struct {
request *http.Request
base string
}
func TestBuilderFromRequest(t *testing.T) {
u, err := url.Parse("http://example.com")
if err != nil {
t.Fatal(err)
}
forwardedProtoHeader := make(http.Header, 1)
forwardedProtoHeader.Set("X-Forwarded-Proto", "https")
forwardedHostHeader1 := make(http.Header, 1)
forwardedHostHeader1.Set("X-Forwarded-Host", "first.example.com")
forwardedHostHeader2 := make(http.Header, 1)
forwardedHostHeader2.Set("X-Forwarded-Host", "first.example.com, proxy1.example.com")
testRequests := []struct {
request *http.Request
base string
configHost url.URL
}{
{
request: &http.Request{URL: u, Host: u.Host},
base: "http://example.com",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader},
base: "http://example.com",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader},
base: "https://example.com",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedHostHeader1},
base: "http://first.example.com",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedHostHeader2},
base: "http://first.example.com",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedHostHeader2},
base: "https://third.example.com:5000",
configHost: url.URL{
Scheme: "https",
Host: "third.example.com:5000",
},
},
}
doTest := func(relative bool) {
for _, tr := range testRequests {
var builder *URLBuilder
if tr.configHost.Scheme != "" && tr.configHost.Host != "" {
builder = NewURLBuilder(&tr.configHost, relative)
} else {
builder = NewURLBuilderFromRequest(tr.request, relative)
}
for _, testCase := range makeURLBuilderTestCases(builder) {
buildURL, err := testCase.build()
if err != nil {
t.Fatalf("%s: error building url: %v", testCase.description, err)
}
var expectedURL string
proto, ok := tr.request.Header["X-Forwarded-Proto"]
if !ok {
expectedURL = testCase.expectedPath
if !relative {
expectedURL = tr.base + expectedURL
}
} else {
urlBase, err := url.Parse(tr.base)
if err != nil {
t.Fatal(err)
}
urlBase.Scheme = proto[0]
expectedURL = testCase.expectedPath
if !relative {
expectedURL = urlBase.String() + expectedURL
}
}
if buildURL != expectedURL {
t.Fatalf("%s: %q != %q", testCase.description, buildURL, expectedURL)
}
}
}
}
doTest(true)
doTest(false)
}
func TestBuilderFromRequestWithPrefix(t *testing.T) {
u, err := url.Parse("http://example.com/prefix/v2/")
if err != nil {
t.Fatal(err)
}
forwardedProtoHeader := make(http.Header, 1)
forwardedProtoHeader.Set("X-Forwarded-Proto", "https")
testRequests := []struct {
request *http.Request
base string
configHost url.URL
}{
{
request: &http.Request{URL: u, Host: u.Host},
base: "http://example.com/prefix/",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader},
base: "http://example.com/prefix/",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader},
base: "https://example.com/prefix/",
},
{
request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader},
base: "https://subdomain.example.com/prefix/",
configHost: url.URL{
Scheme: "https",
Host: "subdomain.example.com",
Path: "/prefix/",
},
},
}
var relative bool
for _, tr := range testRequests {
var builder *URLBuilder
if tr.configHost.Scheme != "" && tr.configHost.Host != "" {
builder = NewURLBuilder(&tr.configHost, false)
} else {
builder = NewURLBuilderFromRequest(tr.request, false)
}
for _, testCase := range makeURLBuilderTestCases(builder) {
buildURL, err := testCase.build()
if err != nil {
t.Fatalf("%s: error building url: %v", testCase.description, err)
}
var expectedURL string
proto, ok := tr.request.Header["X-Forwarded-Proto"]
if !ok {
expectedURL = testCase.expectedPath
if !relative {
expectedURL = tr.base[0:len(tr.base)-1] + expectedURL
}
} else {
urlBase, err := url.Parse(tr.base)
if err != nil {
t.Fatal(err)
}
urlBase.Scheme = proto[0]
expectedURL = testCase.expectedPath
if !relative {
expectedURL = urlBase.String()[0:len(urlBase.String())-1] + expectedURL
}
}
if buildURL != expectedURL {
t.Fatalf("%s: %q != %q", testCase.description, buildURL, expectedURL)
}
}
}
}

54
docs/architecture.md Normal file
View file

@ -0,0 +1,54 @@
<!--[metadata]>
+++
draft = true
+++
<![end-metadata]-->
# Architecture
## Design
**TODO(stevvooe):** Discuss the architecture of the registry, internally and externally, in a few different deployment scenarios.
### Eventual Consistency
> **NOTE:** This section belongs somewhere, perhaps in a design document. We
> are leaving this here so the information is not lost.
Running the registry on eventually consistent backends has been part of the
design from the beginning. This section covers some of the approaches to
dealing with this reality.
There are a few classes of issues that we need to worry about when
implementing something on top of the storage drivers:
1. Read-After-Write consistency (see this [article on
s3](http://shlomoswidler.com/2009/12/read-after-write-consistency-in-amazon.html)).
2. [Write-Write Conflicts](http://en.wikipedia.org/wiki/Write%E2%80%93write_conflict).
In reality, the registry must worry about these kinds of errors when doing the
following:
1. Accepting data into a temporary upload file may not have latest data block
yet (read-after-write).
2. Moving uploaded data into its blob location (write-write race).
3. Modifying the "current" manifest for given tag (write-write race).
4. A whole slew of operations around deletes (read-after-write, delete-write
races, garbage collection, etc.).
The backend path layout employs a few techniques to avoid these problems:
1. Large writes are done to private upload directories. This alleviates most
of the corruption potential under multiple writers by avoiding multiple
writers.
2. Constraints in storage driver implementations, such as support for writing
after the end of a file to extend it.
3. Digest verification to avoid data corruption.
4. Manifest files are stored by digest and cannot change.
5. All other non-content files (links, hashes, etc.) are written as an atomic
unit. Anything that requires additions and deletions is broken out into
separate "files". Last writer still wins.
Unfortunately, one must play this game when trying to build something like
this on top of eventually consistent storage systems. If we run into serious
problems, we can wrap the storagedrivers in a shared consistency layer but
that would increase complexity and hinder registry cluster performance.

168
docs/auth/auth.go
View file

@ -1,168 +0,0 @@
// Package auth defines a standard interface for request access controllers.
//
// An access controller has a simple interface with a single `Authorized`
// method which checks that a given request is authorized to perform one or
// more actions on one or more resources. This method should return a non-nil
// error if the request is not authorized.
//
// An implementation registers its access controller by name with a constructor
// which accepts an options map for configuring the access controller.
//
// options := map[string]interface{}{"sillySecret": "whysosilly?"}
// accessController, _ := auth.GetAccessController("silly", options)
//
// This `accessController` can then be used in a request handler like so:
//
// func updateOrder(w http.ResponseWriter, r *http.Request) {
// orderNumber := r.FormValue("orderNumber")
// resource := auth.Resource{Type: "customerOrder", Name: orderNumber}
// access := auth.Access{Resource: resource, Action: "update"}
//
// if ctx, err := accessController.Authorized(ctx, access); err != nil {
// if challenge, ok := err.(auth.Challenge) {
// // Let the challenge write the response.
// challenge.SetHeaders(w)
// w.WriteHeader(http.StatusUnauthorized)
// return
// } else {
// // Some other error.
// }
// }
// }
//
package auth
import (
"errors"
"fmt"
"net/http"
"github.com/docker/distribution/context"
)
const (
// UserKey is used to get the user object from
// a user context
UserKey = "auth.user"
// UserNameKey is used to get the user name from
// a user context
UserNameKey = "auth.user.name"
)
var (
// ErrInvalidCredential is returned when the auth token does not authenticate correctly.
ErrInvalidCredential = errors.New("invalid authorization credential")
// ErrAuthenticationFailure returned when authentication fails.
ErrAuthenticationFailure = errors.New("authentication failure")
)
// UserInfo carries information about
// an autenticated/authorized client.
type UserInfo struct {
Name string
}
// Resource describes a resource by type and name.
type Resource struct {
Type string
Name string
}
// Access describes a specific action that is
// requested or allowed for a given resource.
type Access struct {
Resource
Action string
}
// Challenge is a special error type which is used for HTTP 401 Unauthorized
// responses and is able to write the response with WWW-Authenticate challenge
// header values based on the error.
type Challenge interface {
error
// SetHeaders prepares the request to conduct a challenge response by
// adding the an HTTP challenge header on the response message. Callers
// are expected to set the appropriate HTTP status code (e.g. 401)
// themselves.
SetHeaders(w http.ResponseWriter)
}
// AccessController controls access to registry resources based on a request
// and required access levels for a request. Implementations can support both
// complete denial and http authorization challenges.
type AccessController interface {
// Authorized returns a non-nil error if the context is granted access and
// returns a new authorized context. If one or more Access structs are
// provided, the requested access will be compared with what is available
// to the context. The given context will contain a "http.request" key with
// a `*http.Request` value. If the error is non-nil, access should always
// be denied. The error may be of type Challenge, in which case the caller
// may have the Challenge handle the request or choose what action to take
// based on the Challenge header or response status. The returned context
// object should have a "auth.user" value set to a UserInfo struct.
Authorized(ctx context.Context, access ...Access) (context.Context, error)
}
// CredentialAuthenticator is an object which is able to authenticate credentials
type CredentialAuthenticator interface {
AuthenticateUser(username, password string) error
}
// WithUser returns a context with the authorized user info.
func WithUser(ctx context.Context, user UserInfo) context.Context {
return userInfoContext{
Context: ctx,
user: user,
}
}
type userInfoContext struct {
context.Context
user UserInfo
}
func (uic userInfoContext) Value(key interface{}) interface{} {
switch key {
case UserKey:
return uic.user
case UserNameKey:
return uic.user.Name
}
return uic.Context.Value(key)
}
// InitFunc is the type of an AccessController factory function and is used
// to register the constructor for different AccesController backends.
type InitFunc func(options map[string]interface{}) (AccessController, error)
var accessControllers map[string]InitFunc
func init() {
accessControllers = make(map[string]InitFunc)
}
// Register is used to register an InitFunc for
// an AccessController backend with the given name.
func Register(name string, initFunc InitFunc) error {
if _, exists := accessControllers[name]; exists {
return fmt.Errorf("name already registered: %s", name)
}
accessControllers[name] = initFunc
return nil
}
// GetAccessController constructs an AccessController
// with the given options using the named backend.
func GetAccessController(name string, options map[string]interface{}) (AccessController, error) {
if initFunc, exists := accessControllers[name]; exists {
return initFunc(options)
}
return nil, fmt.Errorf("no access controller registered with name: %s", name)
}

97
docs/auth/htpasswd/access.go
View file

@ -1,97 +0,0 @@
// Package htpasswd provides a simple authentication scheme that checks for the
// user credential hash in an htpasswd formatted file in a configuration-determined
// location.
//
// This authentication method MUST be used under TLS, as simple token-replay attack is possible.
package htpasswd
import (
"fmt"
"net/http"
"os"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
)
type accessController struct {
realm string
htpasswd *htpasswd
}
var _ auth.AccessController = &accessController{}
func newAccessController(options map[string]interface{}) (auth.AccessController, error) {
realm, present := options["realm"]
if _, ok := realm.(string); !present || !ok {
return nil, fmt.Errorf(`"realm" must be set for htpasswd access controller`)
}
path, present := options["path"]
if _, ok := path.(string); !present || !ok {
return nil, fmt.Errorf(`"path" must be set for htpasswd access controller`)
}
f, err := os.Open(path.(string))
if err != nil {
return nil, err
}
defer f.Close()
h, err := newHTPasswd(f)
if err != nil {
return nil, err
}
return &accessController{realm: realm.(string), htpasswd: h}, nil
}
func (ac *accessController) Authorized(ctx context.Context, accessRecords ...auth.Access) (context.Context, error) {
req, err := context.GetRequest(ctx)
if err != nil {
return nil, err
}
username, password, ok := req.BasicAuth()
if !ok {
return nil, &challenge{
realm: ac.realm,
err: auth.ErrInvalidCredential,
}
}
if err := ac.AuthenticateUser(username, password); err != nil {
context.GetLogger(ctx).Errorf("error authenticating user %q: %v", username, err)
return nil, &challenge{
realm: ac.realm,
err: auth.ErrAuthenticationFailure,
}
}
return auth.WithUser(ctx, auth.UserInfo{Name: username}), nil
}
func (ac *accessController) AuthenticateUser(username, password string) error {
return ac.htpasswd.authenticateUser(username, password)
}
// challenge implements the auth.Challenge interface.
type challenge struct {
realm string
err error
}
var _ auth.Challenge = challenge{}
// SetHeaders sets the basic challenge header on the response.
func (ch challenge) SetHeaders(w http.ResponseWriter) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", ch.realm))
}
func (ch challenge) Error() string {
return fmt.Sprintf("basic authentication challenge for realm %q: %s", ch.realm, ch.err)
}
func init() {
auth.Register("htpasswd", auth.InitFunc(newAccessController))
}

122
docs/auth/htpasswd/access_test.go
View file

@ -1,122 +0,0 @@
package htpasswd
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
)
func TestBasicAccessController(t *testing.T) {
testRealm := "The-Shire"
testUsers := []string{"bilbo", "frodo", "MiShil", "DeokMan"}
testPasswords := []string{"baggins", "baggins", "새주", "공주님"}
testHtpasswdContent := `bilbo:{SHA}5siv5c0SHx681xU6GiSx9ZQryqs=
frodo:$2y$05$926C3y10Quzn/LnqQH86VOEVh/18T6RnLaS.khre96jLNL/7e.K5W
MiShil:$2y$05$0oHgwMehvoe8iAWS8I.7l.KoECXrwVaC16RPfaSCU5eVTFrATuMI2
DeokMan:공주님`
tempFile, err := ioutil.TempFile("", "htpasswd-test")
if err != nil {
t.Fatal("could not create temporary htpasswd file")
}
if _, err = tempFile.WriteString(testHtpasswdContent); err != nil {
t.Fatal("could not write temporary htpasswd file")
}
options := map[string]interface{}{
"realm": testRealm,
"path": tempFile.Name(),
}
ctx := context.Background()
accessController, err := newAccessController(options)
if err != nil {
t.Fatal("error creating access controller")
}
tempFile.Close()
var userNumber = 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := context.WithRequest(ctx, r)
authCtx, err := accessController.Authorized(ctx)
if err != nil {
switch err := err.(type) {
case auth.Challenge:
err.SetHeaders(w)
w.WriteHeader(http.StatusUnauthorized)
return
default:
t.Fatalf("unexpected error authorizing request: %v", err)
}
}
userInfo, ok := authCtx.Value(auth.UserKey).(auth.UserInfo)
if !ok {
t.Fatal("basic accessController did not set auth.user context")
}
if userInfo.Name != testUsers[userNumber] {
t.Fatalf("expected user name %q, got %q", testUsers[userNumber], userInfo.Name)
}
w.WriteHeader(http.StatusNoContent)
}))
client := &http.Client{
CheckRedirect: nil,
}
req, _ := http.NewRequest("GET", server.URL, nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("unexpected error during GET: %v", err)
}
defer resp.Body.Close()
// Request should not be authorized
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("unexpected non-fail response status: %v != %v", resp.StatusCode, http.StatusUnauthorized)
}
nonbcrypt := map[string]struct{}{
"bilbo": {},
"DeokMan": {},
}
for i := 0; i < len(testUsers); i++ {
userNumber = i
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
t.Fatalf("error allocating new request: %v", err)
}
req.SetBasicAuth(testUsers[i], testPasswords[i])
resp, err = client.Do(req)
if err != nil {
t.Fatalf("unexpected error during GET: %v", err)
}
defer resp.Body.Close()
if _, ok := nonbcrypt[testUsers[i]]; ok {
// these are not allowed.
// Request should be authorized
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("unexpected non-success response status: %v != %v for %s %s", resp.StatusCode, http.StatusUnauthorized, testUsers[i], testPasswords[i])
}
} else {
// Request should be authorized
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unexpected non-success response status: %v != %v for %s %s", resp.StatusCode, http.StatusNoContent, testUsers[i], testPasswords[i])
}
}
}
}

82
docs/auth/htpasswd/htpasswd.go
View file

@ -1,82 +0,0 @@
package htpasswd
import (
"bufio"
"fmt"
"io"
"strings"
"github.com/docker/distribution/registry/auth"
"golang.org/x/crypto/bcrypt"
)
// htpasswd holds a path to a system .htpasswd file and the machinery to parse
// it. Only bcrypt hash entries are supported.
type htpasswd struct {
entries map[string][]byte // maps username to password byte slice.
}
// newHTPasswd parses the reader and returns an htpasswd or an error.
func newHTPasswd(rd io.Reader) (*htpasswd, error) {
entries, err := parseHTPasswd(rd)
if err != nil {
return nil, err
}
return &htpasswd{entries: entries}, nil
}
// AuthenticateUser checks a given user:password credential against the
// receiving HTPasswd's file. If the check passes, nil is returned.
func (htpasswd *htpasswd) authenticateUser(username string, password string) error {
credentials, ok := htpasswd.entries[username]
if !ok {
// timing attack paranoia
bcrypt.CompareHashAndPassword([]byte{}, []byte(password))
return auth.ErrAuthenticationFailure
}
err := bcrypt.CompareHashAndPassword([]byte(credentials), []byte(password))
if err != nil {
return auth.ErrAuthenticationFailure
}
return nil
}
// parseHTPasswd parses the contents of htpasswd. This will read all the
// entries in the file, whether or not they are needed. An error is returned
// if a syntax errors are encountered or if the reader fails.
func parseHTPasswd(rd io.Reader) (map[string][]byte, error) {
entries := map[string][]byte{}
scanner := bufio.NewScanner(rd)
var line int
for scanner.Scan() {
line++ // 1-based line numbering
t := strings.TrimSpace(scanner.Text())
if len(t) < 1 {
continue
}
// lines that *begin* with a '#' are considered comments
if t[0] == '#' {
continue
}
i := strings.Index(t, ":")
if i < 0 || i >= len(t) {
return nil, fmt.Errorf("htpasswd: invalid entry at line %d: %q", line, scanner.Text())
}
entries[t[:i]] = []byte(t[i+1:])
}
if err := scanner.Err(); err != nil {
return nil, err
}
return entries, nil
}

85
docs/auth/htpasswd/htpasswd_test.go
View file

@ -1,85 +0,0 @@
package htpasswd
import (
"fmt"
"reflect"
"strings"
"testing"
)
func TestParseHTPasswd(t *testing.T) {
for _, tc := range []struct {
desc string
input string
err error
entries map[string][]byte
}{
{
desc: "basic example",
input: `
# This is a comment in a basic example.
bilbo:{SHA}5siv5c0SHx681xU6GiSx9ZQryqs=
frodo:$2y$05$926C3y10Quzn/LnqQH86VOEVh/18T6RnLaS.khre96jLNL/7e.K5W
MiShil:$2y$05$0oHgwMehvoe8iAWS8I.7l.KoECXrwVaC16RPfaSCU5eVTFrATuMI2
DeokMan:공주님
`,
entries: map[string][]byte{
"bilbo": []byte("{SHA}5siv5c0SHx681xU6GiSx9ZQryqs="),
"frodo": []byte("$2y$05$926C3y10Quzn/LnqQH86VOEVh/18T6RnLaS.khre96jLNL/7e.K5W"),
"MiShil": []byte("$2y$05$0oHgwMehvoe8iAWS8I.7l.KoECXrwVaC16RPfaSCU5eVTFrATuMI2"),
"DeokMan": []byte("공주님"),
},
},
{
desc: "ensures comments are filtered",
input: `
# asdf:asdf
`,
},
{
desc: "ensure midline hash is not comment",
input: `
asdf:as#df
`,
entries: map[string][]byte{
"asdf": []byte("as#df"),
},
},
{
desc: "ensure midline hash is not comment",
input: `
# A valid comment
valid:entry
asdf
`,
err: fmt.Errorf(`htpasswd: invalid entry at line 4: "asdf"`),
},
} {
entries, err := parseHTPasswd(strings.NewReader(tc.input))
if err != tc.err {
if tc.err == nil {
t.Fatalf("%s: unexpected error: %v", tc.desc, err)
} else {
if err.Error() != tc.err.Error() { // use string equality here.
t.Fatalf("%s: expected error not returned: %v != %v", tc.desc, err, tc.err)
}
}
}
if tc.err != nil {
continue // don't test output
}
// allow empty and nil to be equal
if tc.entries == nil {
tc.entries = map[string][]byte{}
}
if !reflect.DeepEqual(entries, tc.entries) {
t.Fatalf("%s: entries not parsed correctly: %v != %v", tc.desc, entries, tc.entries)
}
}
}

97
docs/auth/silly/access.go
View file

@ -1,97 +0,0 @@
// Package silly provides a simple authentication scheme that checks for the
// existence of an Authorization header and issues access if is present and
// non-empty.
//
// This package is present as an example implementation of a minimal
// auth.AccessController and for testing. This is not suitable for any kind of
// production security.
package silly
import (
"fmt"
"net/http"
"strings"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
)
// accessController provides a simple implementation of auth.AccessController
// that simply checks for a non-empty Authorization header. It is useful for
// demonstration and testing.
type accessController struct {
realm string
service string
}
var _ auth.AccessController = &accessController{}
func newAccessController(options map[string]interface{}) (auth.AccessController, error) {
realm, present := options["realm"]
if _, ok := realm.(string); !present || !ok {
return nil, fmt.Errorf(`"realm" must be set for silly access controller`)
}
service, present := options["service"]
if _, ok := service.(string); !present || !ok {
return nil, fmt.Errorf(`"service" must be set for silly access controller`)
}
return &accessController{realm: realm.(string), service: service.(string)}, nil
}
// Authorized simply checks for the existence of the authorization header,
// responding with a bearer challenge if it doesn't exist.
func (ac *accessController) Authorized(ctx context.Context, accessRecords ...auth.Access) (context.Context, error) {
req, err := context.GetRequest(ctx)
if err != nil {
return nil, err
}
if req.Header.Get("Authorization") == "" {
challenge := challenge{
realm: ac.realm,
service: ac.service,
}
if len(accessRecords) > 0 {
var scopes []string
for _, access := range accessRecords {
scopes = append(scopes, fmt.Sprintf("%s:%s:%s", access.Type, access.Resource.Name, access.Action))
}
challenge.scope = strings.Join(scopes, " ")
}
return nil, &challenge
}
return auth.WithUser(ctx, auth.UserInfo{Name: "silly"}), nil
}
type challenge struct {
realm string
service string
scope string
}
var _ auth.Challenge = challenge{}
// SetHeaders sets a simple bearer challenge on the response.
func (ch challenge) SetHeaders(w http.ResponseWriter) {
header := fmt.Sprintf("Bearer realm=%q,service=%q", ch.realm, ch.service)
if ch.scope != "" {
header = fmt.Sprintf("%s,scope=%q", header, ch.scope)
}
w.Header().Set("WWW-Authenticate", header)
}
func (ch challenge) Error() string {
return fmt.Sprintf("silly authentication challenge: %#v", ch)
}
// init registers the silly auth backend.
func init() {
auth.Register("silly", auth.InitFunc(newAccessController))
}

71
docs/auth/silly/access_test.go
View file

@ -1,71 +0,0 @@
package silly
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
)
func TestSillyAccessController(t *testing.T) {
ac := &accessController{
realm: "test-realm",
service: "test-service",
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := context.WithValue(nil, "http.request", r)
authCtx, err := ac.Authorized(ctx)
if err != nil {
switch err := err.(type) {
case auth.Challenge:
err.SetHeaders(w)
w.WriteHeader(http.StatusUnauthorized)
return
default:
t.Fatalf("unexpected error authorizing request: %v", err)
}
}
userInfo, ok := authCtx.Value(auth.UserKey).(auth.UserInfo)
if !ok {
t.Fatal("silly accessController did not set auth.user context")
}
if userInfo.Name != "silly" {
t.Fatalf("expected user name %q, got %q", "silly", userInfo.Name)
}
w.WriteHeader(http.StatusNoContent)
}))
resp, err := http.Get(server.URL)
if err != nil {
t.Fatalf("unexpected error during GET: %v", err)
}
defer resp.Body.Close()
// Request should not be authorized
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("unexpected response status: %v != %v", resp.StatusCode, http.StatusUnauthorized)
}
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
t.Fatalf("unexpected error creating new request: %v", err)
}
req.Header.Set("Authorization", "seriously, anything")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unexpected error during GET: %v", err)
}
defer resp.Body.Close()
// Request should not be authorized
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unexpected response status: %v != %v", resp.StatusCode, http.StatusNoContent)
}
}

268
docs/auth/token/accesscontroller.go
View file

@ -1,268 +0,0 @@
package token
import (
"crypto"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
"github.com/docker/libtrust"
)
// accessSet maps a typed, named resource to
// a set of actions requested or authorized.
type accessSet map[auth.Resource]actionSet
// newAccessSet constructs an accessSet from
// a variable number of auth.Access items.
func newAccessSet(accessItems ...auth.Access) accessSet {
accessSet := make(accessSet, len(accessItems))
for _, access := range accessItems {
resource := auth.Resource{
Type: access.Type,
Name: access.Name,
}
set, exists := accessSet[resource]
if !exists {
set = newActionSet()
accessSet[resource] = set
}
set.add(access.Action)
}
return accessSet
}
// contains returns whether or not the given access is in this accessSet.
func (s accessSet) contains(access auth.Access) bool {
actionSet, ok := s[access.Resource]
if ok {
return actionSet.contains(access.Action)
}
return false
}
// scopeParam returns a collection of scopes which can
// be used for a WWW-Authenticate challenge parameter.
// See https://tools.ietf.org/html/rfc6750#section-3
func (s accessSet) scopeParam() string {
scopes := make([]string, 0, len(s))
for resource, actionSet := range s {
actions := strings.Join(actionSet.keys(), ",")
scopes = append(scopes, fmt.Sprintf("%s:%s:%s", resource.Type, resource.Name, actions))
}
return strings.Join(scopes, " ")
}
// Errors used and exported by this package.
var (
ErrInsufficientScope = errors.New("insufficient scope")
ErrTokenRequired = errors.New("authorization token required")
)
// authChallenge implements the auth.Challenge interface.
type authChallenge struct {
err error
realm string
service string
accessSet accessSet
}
var _ auth.Challenge = authChallenge{}
// Error returns the internal error string for this authChallenge.
func (ac authChallenge) Error() string {
return ac.err.Error()
}
// Status returns the HTTP Response Status Code for this authChallenge.
func (ac authChallenge) Status() int {
return http.StatusUnauthorized
}
// challengeParams constructs the value to be used in
// the WWW-Authenticate response challenge header.
// See https://tools.ietf.org/html/rfc6750#section-3
func (ac authChallenge) challengeParams() string {
str := fmt.Sprintf("Bearer realm=%q,service=%q", ac.realm, ac.service)
if scope := ac.accessSet.scopeParam(); scope != "" {
str = fmt.Sprintf("%s,scope=%q", str, scope)
}
if ac.err == ErrInvalidToken || ac.err == ErrMalformedToken {
str = fmt.Sprintf("%s,error=%q", str, "invalid_token")
} else if ac.err == ErrInsufficientScope {
str = fmt.Sprintf("%s,error=%q", str, "insufficient_scope")
}
return str
}
// SetChallenge sets the WWW-Authenticate value for the response.
func (ac authChallenge) SetHeaders(w http.ResponseWriter) {
w.Header().Add("WWW-Authenticate", ac.challengeParams())
}
// accessController implements the auth.AccessController interface.
type accessController struct {
realm string
issuer string
service string
rootCerts *x509.CertPool
trustedKeys map[string]libtrust.PublicKey
}
// tokenAccessOptions is a convenience type for handling
// options to the contstructor of an accessController.
type tokenAccessOptions struct {
realm string
issuer string
service string
rootCertBundle string
}
// checkOptions gathers the necessary options
// for an accessController from the given map.
func checkOptions(options map[string]interface{}) (tokenAccessOptions, error) {
var opts tokenAccessOptions
keys := []string{"realm", "issuer", "service", "rootcertbundle"}
vals := make([]string, 0, len(keys))
for _, key := range keys {
val, ok := options[key].(string)
if !ok {
return opts, fmt.Errorf("token auth requires a valid option string: %q", key)
}
vals = append(vals, val)
}
opts.realm, opts.issuer, opts.service, opts.rootCertBundle = vals[0], vals[1], vals[2], vals[3]
return opts, nil
}
// newAccessController creates an accessController using the given options.
func newAccessController(options map[string]interface{}) (auth.AccessController, error) {
config, err := checkOptions(options)
if err != nil {
return nil, err
}
fp, err := os.Open(config.rootCertBundle)
if err != nil {
return nil, fmt.Errorf("unable to open token auth root certificate bundle file %q: %s", config.rootCertBundle, err)
}
defer fp.Close()
rawCertBundle, err := ioutil.ReadAll(fp)
if err != nil {
return nil, fmt.Errorf("unable to read token auth root certificate bundle file %q: %s", config.rootCertBundle, err)
}
var rootCerts []*x509.Certificate
pemBlock, rawCertBundle := pem.Decode(rawCertBundle)
for pemBlock != nil {
cert, err := x509.ParseCertificate(pemBlock.Bytes)
if err != nil {
return nil, fmt.Errorf("unable to parse token auth root certificate: %s", err)
}
rootCerts = append(rootCerts, cert)
pemBlock, rawCertBundle = pem.Decode(rawCertBundle)
}
if len(rootCerts) == 0 {
return nil, errors.New("token auth requires at least one token signing root certificate")
}
rootPool := x509.NewCertPool()
trustedKeys := make(map[string]libtrust.PublicKey, len(rootCerts))
for _, rootCert := range rootCerts {
rootPool.AddCert(rootCert)
pubKey, err := libtrust.FromCryptoPublicKey(crypto.PublicKey(rootCert.PublicKey))
if err != nil {
return nil, fmt.Errorf("unable to get public key from token auth root certificate: %s", err)
}
trustedKeys[pubKey.KeyID()] = pubKey
}
return &accessController{
realm: config.realm,
issuer: config.issuer,
service: config.service,
rootCerts: rootPool,
trustedKeys: trustedKeys,
}, nil
}
// Authorized handles checking whether the given request is authorized
// for actions on resources described by the given access items.
func (ac *accessController) Authorized(ctx context.Context, accessItems ...auth.Access) (context.Context, error) {
challenge := &authChallenge{
realm: ac.realm,
service: ac.service,
accessSet: newAccessSet(accessItems...),
}
req, err := context.GetRequest(ctx)
if err != nil {
return nil, err
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" {
challenge.err = ErrTokenRequired
return nil, challenge
}
rawToken := parts[1]
token, err := NewToken(rawToken)
if err != nil {
challenge.err = err
return nil, challenge
}
verifyOpts := VerifyOptions{
TrustedIssuers: []string{ac.issuer},
AcceptedAudiences: []string{ac.service},
Roots: ac.rootCerts,
TrustedKeys: ac.trustedKeys,
}
if err = token.Verify(verifyOpts); err != nil {
challenge.err = err
return nil, challenge
}
accessSet := token.accessSet()
for _, access := range accessItems {
if !accessSet.contains(access) {
challenge.err = ErrInsufficientScope
return nil, challenge
}
}
return auth.WithUser(ctx, auth.UserInfo{Name: token.Claims.Subject}), nil
}
// init handles registering the token auth backend.
func init() {
auth.Register("token", auth.InitFunc(newAccessController))
}

35
docs/auth/token/stringset.go
View file

@ -1,35 +0,0 @@
package token
// StringSet is a useful type for looking up strings.
type stringSet map[string]struct{}
// NewStringSet creates a new StringSet with the given strings.
func newStringSet(keys ...string) stringSet {
ss := make(stringSet, len(keys))
ss.add(keys...)
return ss
}
// Add inserts the given keys into this StringSet.
func (ss stringSet) add(keys ...string) {
for _, key := range keys {
ss[key] = struct{}{}
}
}
// Contains returns whether the given key is in this StringSet.
func (ss stringSet) contains(key string) bool {
_, ok := ss[key]
return ok
}
// Keys returns a slice of all keys in this StringSet.
func (ss stringSet) keys() []string {
keys := make([]string, 0, len(ss))
for key := range ss {
keys = append(keys, key)
}
return keys
}

343
docs/auth/token/token.go
View file

@ -1,343 +0,0 @@
package token
import (
"crypto"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"github.com/docker/libtrust"
"github.com/docker/distribution/registry/auth"
)
const (
// TokenSeparator is the value which separates the header, claims, and
// signature in the compact serialization of a JSON Web Token.
TokenSeparator = "."
)
// Errors used by token parsing and verification.
var (
ErrMalformedToken = errors.New("malformed token")
ErrInvalidToken = errors.New("invalid token")
)
// ResourceActions stores allowed actions on a named and typed resource.
type ResourceActions struct {
Type string `json:"type"`
Name string `json:"name"`
Actions []string `json:"actions"`
}
// ClaimSet describes the main section of a JSON Web Token.
type ClaimSet struct {
// Public claims
Issuer string `json:"iss"`
Subject string `json:"sub"`
Audience string `json:"aud"`
Expiration int64 `json:"exp"`
NotBefore int64 `json:"nbf"`
IssuedAt int64 `json:"iat"`
JWTID string `json:"jti"`
// Private claims
Access []*ResourceActions `json:"access"`
}
// Header describes the header section of a JSON Web Token.
type Header struct {
Type string `json:"typ"`
SigningAlg string `json:"alg"`
KeyID string `json:"kid,omitempty"`
X5c []string `json:"x5c,omitempty"`
RawJWK *json.RawMessage `json:"jwk,omitempty"`
}
// Token describes a JSON Web Token.
type Token struct {
Raw string
Header *Header
Claims *ClaimSet
Signature []byte
}
// VerifyOptions is used to specify
// options when verifying a JSON Web Token.
type VerifyOptions struct {
TrustedIssuers []string
AcceptedAudiences []string
Roots *x509.CertPool
TrustedKeys map[string]libtrust.PublicKey
}
// NewToken parses the given raw token string
// and constructs an unverified JSON Web Token.
func NewToken(rawToken string) (*Token, error) {
parts := strings.Split(rawToken, TokenSeparator)
if len(parts) != 3 {
return nil, ErrMalformedToken
}
var (
rawHeader, rawClaims = parts[0], parts[1]
headerJSON, claimsJSON []byte
err error
)
defer func() {
if err != nil {
log.Errorf("error while unmarshalling raw token: %s", err)
}
}()
if headerJSON, err = joseBase64UrlDecode(rawHeader); err != nil {
err = fmt.Errorf("unable to decode header: %s", err)
return nil, ErrMalformedToken
}
if claimsJSON, err = joseBase64UrlDecode(rawClaims); err != nil {
err = fmt.Errorf("unable to decode claims: %s", err)
return nil, ErrMalformedToken
}
token := new(Token)
token.Header = new(Header)
token.Claims = new(ClaimSet)
token.Raw = strings.Join(parts[:2], TokenSeparator)
if token.Signature, err = joseBase64UrlDecode(parts[2]); err != nil {
err = fmt.Errorf("unable to decode signature: %s", err)
return nil, ErrMalformedToken
}
if err = json.Unmarshal(headerJSON, token.Header); err != nil {
return nil, ErrMalformedToken
}
if err = json.Unmarshal(claimsJSON, token.Claims); err != nil {
return nil, ErrMalformedToken
}
return token, nil
}
// Verify attempts to verify this token using the given options.
// Returns a nil error if the token is valid.
func (t *Token) Verify(verifyOpts VerifyOptions) error {
// Verify that the Issuer claim is a trusted authority.
if !contains(verifyOpts.TrustedIssuers, t.Claims.Issuer) {
log.Errorf("token from untrusted issuer: %q", t.Claims.Issuer)
return ErrInvalidToken
}
// Verify that the Audience claim is allowed.
if !contains(verifyOpts.AcceptedAudiences, t.Claims.Audience) {
log.Errorf("token intended for another audience: %q", t.Claims.Audience)
return ErrInvalidToken
}
// Verify that the token is currently usable and not expired.
currentUnixTime := time.Now().Unix()
if !(t.Claims.NotBefore <= currentUnixTime && currentUnixTime <= t.Claims.Expiration) {
log.Errorf("token not to be used before %d or after %d - currently %d", t.Claims.NotBefore, t.Claims.Expiration, currentUnixTime)
return ErrInvalidToken
}
// Verify the token signature.
if len(t.Signature) == 0 {
log.Error("token has no signature")
return ErrInvalidToken
}
// Verify that the signing key is trusted.
signingKey, err := t.VerifySigningKey(verifyOpts)
if err != nil {
log.Error(err)
return ErrInvalidToken
}
// Finally, verify the signature of the token using the key which signed it.
if err := signingKey.Verify(strings.NewReader(t.Raw), t.Header.SigningAlg, t.Signature); err != nil {
log.Errorf("unable to verify token signature: %s", err)
return ErrInvalidToken
}
return nil
}
// VerifySigningKey attempts to get the key which was used to sign this token.
// The token header should contain either of these 3 fields:
// `x5c` - The x509 certificate chain for the signing key. Needs to be
// verified.
// `jwk` - The JSON Web Key representation of the signing key.
// May contain its own `x5c` field which needs to be verified.
// `kid` - The unique identifier for the key. This library interprets it
// as a libtrust fingerprint. The key itself can be looked up in
// the trustedKeys field of the given verify options.
// Each of these methods are tried in that order of preference until the
// signing key is found or an error is returned.
func (t *Token) VerifySigningKey(verifyOpts VerifyOptions) (signingKey libtrust.PublicKey, err error) {
// First attempt to get an x509 certificate chain from the header.
var (
x5c = t.Header.X5c
rawJWK = t.Header.RawJWK
keyID = t.Header.KeyID
)
switch {
case len(x5c) > 0:
signingKey, err = parseAndVerifyCertChain(x5c, verifyOpts.Roots)
case rawJWK != nil:
signingKey, err = parseAndVerifyRawJWK(rawJWK, verifyOpts)
case len(keyID) > 0:
signingKey = verifyOpts.TrustedKeys[keyID]
if signingKey == nil {
err = fmt.Errorf("token signed by untrusted key with ID: %q", keyID)
}
default:
err = errors.New("unable to get token signing key")
}
return
}
func parseAndVerifyCertChain(x5c []string, roots *x509.CertPool) (leafKey libtrust.PublicKey, err error) {
if len(x5c) == 0 {
return nil, errors.New("empty x509 certificate chain")
}
// Ensure the first element is encoded correctly.
leafCertDer, err := base64.StdEncoding.DecodeString(x5c[0])
if err != nil {
return nil, fmt.Errorf("unable to decode leaf certificate: %s", err)
}
// And that it is a valid x509 certificate.
leafCert, err := x509.ParseCertificate(leafCertDer)
if err != nil {
return nil, fmt.Errorf("unable to parse leaf certificate: %s", err)
}
// The rest of the certificate chain are intermediate certificates.
intermediates := x509.NewCertPool()
for i := 1; i < len(x5c); i++ {
intermediateCertDer, err := base64.StdEncoding.DecodeString(x5c[i])
if err != nil {
return nil, fmt.Errorf("unable to decode intermediate certificate: %s", err)
}
intermediateCert, err := x509.ParseCertificate(intermediateCertDer)
if err != nil {
return nil, fmt.Errorf("unable to parse intermediate certificate: %s", err)
}
intermediates.AddCert(intermediateCert)
}
verifyOpts := x509.VerifyOptions{
Intermediates: intermediates,
Roots: roots,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
}
// TODO: this call returns certificate chains which we ignore for now, but
// we should check them for revocations if we have the ability later.
if _, err = leafCert.Verify(verifyOpts); err != nil {
return nil, fmt.Errorf("unable to verify certificate chain: %s", err)
}
// Get the public key from the leaf certificate.
leafCryptoKey, ok := leafCert.PublicKey.(crypto.PublicKey)
if !ok {
return nil, errors.New("unable to get leaf cert public key value")
}
leafKey, err = libtrust.FromCryptoPublicKey(leafCryptoKey)
if err != nil {
return nil, fmt.Errorf("unable to make libtrust public key from leaf certificate: %s", err)
}
return
}
func parseAndVerifyRawJWK(rawJWK *json.RawMessage, verifyOpts VerifyOptions) (pubKey libtrust.PublicKey, err error) {
pubKey, err = libtrust.UnmarshalPublicKeyJWK([]byte(*rawJWK))
if err != nil {
return nil, fmt.Errorf("unable to decode raw JWK value: %s", err)
}
// Check to see if the key includes a certificate chain.
x5cVal, ok := pubKey.GetExtendedField("x5c").([]interface{})
if !ok {
// The JWK should be one of the trusted root keys.
if _, trusted := verifyOpts.TrustedKeys[pubKey.KeyID()]; !trusted {
return nil, errors.New("untrusted JWK with no certificate chain")
}
// The JWK is one of the trusted keys.
return
}
// Ensure each item in the chain is of the correct type.
x5c := make([]string, len(x5cVal))
for i, val := range x5cVal {
certString, ok := val.(string)
if !ok || len(certString) == 0 {
return nil, errors.New("malformed certificate chain")
}
x5c[i] = certString
}
// Ensure that the x509 certificate chain can
// be verified up to one of our trusted roots.
leafKey, err := parseAndVerifyCertChain(x5c, verifyOpts.Roots)
if err != nil {
return nil, fmt.Errorf("could not verify JWK certificate chain: %s", err)
}
// Verify that the public key in the leaf cert *is* the signing key.
if pubKey.KeyID() != leafKey.KeyID() {
return nil, errors.New("leaf certificate public key ID does not match JWK key ID")
}
return
}
// accessSet returns a set of actions available for the resource
// actions listed in the `access` section of this token.
func (t *Token) accessSet() accessSet {
if t.Claims == nil {
return nil
}
accessSet := make(accessSet, len(t.Claims.Access))
for _, resourceActions := range t.Claims.Access {
resource := auth.Resource{
Type: resourceActions.Type,
Name: resourceActions.Name,
}
set, exists := accessSet[resource]
if !exists {
set = newActionSet()
accessSet[resource] = set
}
for _, action := range resourceActions.Actions {
set.add(action)
}
}
return accessSet
}
func (t *Token) compactRaw() string {
return fmt.Sprintf("%s.%s", t.Raw, joseBase64UrlEncode(t.Signature))
}

387
docs/auth/token/token_test.go
View file

@ -1,387 +0,0 @@
package token
import (
"crypto"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/auth"
"github.com/docker/libtrust"
)
func makeRootKeys(numKeys int) ([]libtrust.PrivateKey, error) {
keys := make([]libtrust.PrivateKey, 0, numKeys)
for i := 0; i < numKeys; i++ {
key, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
return nil, err
}
keys = append(keys, key)
}
return keys, nil
}
func makeSigningKeyWithChain(rootKey libtrust.PrivateKey, depth int) (libtrust.PrivateKey, error) {
if depth == 0 {
// Don't need to build a chain.
return rootKey, nil
}
var (
x5c = make([]string, depth)
parentKey = rootKey
key libtrust.PrivateKey
cert *x509.Certificate
err error
)
for depth > 0 {
if key, err = libtrust.GenerateECP256PrivateKey(); err != nil {
return nil, err
}
if cert, err = libtrust.GenerateCACert(parentKey, key); err != nil {
return nil, err
}
depth--
x5c[depth] = base64.StdEncoding.EncodeToString(cert.Raw)
parentKey = key
}
key.AddExtendedField("x5c", x5c)
return key, nil
}
func makeRootCerts(rootKeys []libtrust.PrivateKey) ([]*x509.Certificate, error) {
certs := make([]*x509.Certificate, 0, len(rootKeys))
for _, key := range rootKeys {
cert, err := libtrust.GenerateCACert(key, key)
if err != nil {
return nil, err
}
certs = append(certs, cert)
}
return certs, nil
}
func makeTrustedKeyMap(rootKeys []libtrust.PrivateKey) map[string]libtrust.PublicKey {
trustedKeys := make(map[string]libtrust.PublicKey, len(rootKeys))
for _, key := range rootKeys {
trustedKeys[key.KeyID()] = key.PublicKey()
}
return trustedKeys
}
func makeTestToken(issuer, audience string, access []*ResourceActions, rootKey libtrust.PrivateKey, depth int) (*Token, error) {
signingKey, err := makeSigningKeyWithChain(rootKey, depth)
if err != nil {
return nil, fmt.Errorf("unable to make signing key with chain: %s", err)
}
var rawJWK json.RawMessage
rawJWK, err = signingKey.PublicKey().MarshalJSON()
if err != nil {
return nil, fmt.Errorf("unable to marshal signing key to JSON: %s", err)
}
joseHeader := &Header{
Type: "JWT",
SigningAlg: "ES256",
RawJWK: &rawJWK,
}
now := time.Now()
randomBytes := make([]byte, 15)
if _, err = rand.Read(randomBytes); err != nil {
return nil, fmt.Errorf("unable to read random bytes for jwt id: %s", err)
}
claimSet := &ClaimSet{
Issuer: issuer,
Subject: "foo",
Audience: audience,
Expiration: now.Add(5 * time.Minute).Unix(),
NotBefore: now.Unix(),
IssuedAt: now.Unix(),
JWTID: base64.URLEncoding.EncodeToString(randomBytes),
Access: access,
}
var joseHeaderBytes, claimSetBytes []byte
if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
return nil, fmt.Errorf("unable to marshal jose header: %s", err)
}
if claimSetBytes, err = json.Marshal(claimSet); err != nil {
return nil, fmt.Errorf("unable to marshal claim set: %s", err)
}
encodedJoseHeader := joseBase64UrlEncode(joseHeaderBytes)
encodedClaimSet := joseBase64UrlEncode(claimSetBytes)
encodingToSign := fmt.Sprintf("%s.%s", encodedJoseHeader, encodedClaimSet)
var signatureBytes []byte
if signatureBytes, _, err = signingKey.Sign(strings.NewReader(encodingToSign), crypto.SHA256); err != nil {
return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
}
signature := joseBase64UrlEncode(signatureBytes)
tokenString := fmt.Sprintf("%s.%s", encodingToSign, signature)
return NewToken(tokenString)
}
// This test makes 4 tokens with a varying number of intermediate
// certificates ranging from no intermediate chain to a length of 3
// intermediates.
func TestTokenVerify(t *testing.T) {
var (
numTokens = 4
issuer = "test-issuer"
audience = "test-audience"
access = []*ResourceActions{
{
Type: "repository",
Name: "foo/bar",
Actions: []string{"pull", "push"},
},
}
)
rootKeys, err := makeRootKeys(numTokens)
if err != nil {
t.Fatal(err)
}
rootCerts, err := makeRootCerts(rootKeys)
if err != nil {
t.Fatal(err)
}
rootPool := x509.NewCertPool()
for _, rootCert := range rootCerts {
rootPool.AddCert(rootCert)
}
trustedKeys := makeTrustedKeyMap(rootKeys)
tokens := make([]*Token, 0, numTokens)
for i := 0; i < numTokens; i++ {
token, err := makeTestToken(issuer, audience, access, rootKeys[i], i)
if err != nil {
t.Fatal(err)
}
tokens = append(tokens, token)
}
verifyOps := VerifyOptions{
TrustedIssuers: []string{issuer},
AcceptedAudiences: []string{audience},
Roots: rootPool,
TrustedKeys: trustedKeys,
}
for _, token := range tokens {
if err := token.Verify(verifyOps); err != nil {
t.Fatal(err)
}
}
}
func writeTempRootCerts(rootKeys []libtrust.PrivateKey) (filename string, err error) {
rootCerts, err := makeRootCerts(rootKeys)
if err != nil {
return "", err
}
tempFile, err := ioutil.TempFile("", "rootCertBundle")
if err != nil {
return "", err
}
defer tempFile.Close()
for _, cert := range rootCerts {
if err = pem.Encode(tempFile, &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}); err != nil {
os.Remove(tempFile.Name())
return "", err
}
}
return tempFile.Name(), nil
}
// TestAccessController tests complete integration of the token auth package.
// It starts by mocking the options for a token auth accessController which
// it creates. It then tries a few mock requests:
// - don't supply a token; should error with challenge
// - supply an invalid token; should error with challenge
// - supply a token with insufficient access; should error with challenge
// - supply a valid token; should not error
func TestAccessController(t *testing.T) {
// Make 2 keys; only the first is to be a trusted root key.
rootKeys, err := makeRootKeys(2)
if err != nil {
t.Fatal(err)
}
rootCertBundleFilename, err := writeTempRootCerts(rootKeys[:1])
if err != nil {
t.Fatal(err)
}
defer os.Remove(rootCertBundleFilename)
realm := "https://auth.example.com/token/"
issuer := "test-issuer.example.com"
service := "test-service.example.com"
options := map[string]interface{}{
"realm": realm,
"issuer": issuer,
"service": service,
"rootcertbundle": rootCertBundleFilename,
}
accessController, err := newAccessController(options)
if err != nil {
t.Fatal(err)
}
// 1. Make a mock http.Request with no token.
req, err := http.NewRequest("GET", "http://example.com/foo", nil)
if err != nil {
t.Fatal(err)
}
testAccess := auth.Access{
Resource: auth.Resource{
Type: "foo",
Name: "bar",
},
Action: "baz",
}
ctx := context.WithValue(nil, "http.request", req)
authCtx, err := accessController.Authorized(ctx, testAccess)
challenge, ok := err.(auth.Challenge)
if !ok {
t.Fatal("accessController did not return a challenge")
}
if challenge.Error() != ErrTokenRequired.Error() {
t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrTokenRequired)
}
if authCtx != nil {
t.Fatalf("expected nil auth context but got %s", authCtx)
}
// 2. Supply an invalid token.
token, err := makeTestToken(
issuer, service,
[]*ResourceActions{{
Type: testAccess.Type,
Name: testAccess.Name,
Actions: []string{testAccess.Action},
}},
rootKeys[1], 1, // Everything is valid except the key which signed it.
)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw()))
authCtx, err = accessController.Authorized(ctx, testAccess)
challenge, ok = err.(auth.Challenge)
if !ok {
t.Fatal("accessController did not return a challenge")
}
if challenge.Error() != ErrInvalidToken.Error() {
t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrTokenRequired)
}
if authCtx != nil {
t.Fatalf("expected nil auth context but got %s", authCtx)
}
// 3. Supply a token with insufficient access.
token, err = makeTestToken(
issuer, service,
[]*ResourceActions{}, // No access specified.
rootKeys[0], 1,
)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw()))
authCtx, err = accessController.Authorized(ctx, testAccess)
challenge, ok = err.(auth.Challenge)
if !ok {
t.Fatal("accessController did not return a challenge")
}
if challenge.Error() != ErrInsufficientScope.Error() {
t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrInsufficientScope)
}
if authCtx != nil {
t.Fatalf("expected nil auth context but got %s", authCtx)
}
// 4. Supply the token we need, or deserve, or whatever.
token, err = makeTestToken(
issuer, service,
[]*ResourceActions{{
Type: testAccess.Type,
Name: testAccess.Name,
Actions: []string{testAccess.Action},
}},
rootKeys[0], 1,
)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw()))
authCtx, err = accessController.Authorized(ctx, testAccess)
if err != nil {
t.Fatalf("accessController returned unexpected error: %s", err)
}
userInfo, ok := authCtx.Value(auth.UserKey).(auth.UserInfo)
if !ok {
t.Fatal("token accessController did not set auth.user context")
}
if userInfo.Name != "foo" {
t.Fatalf("expected user name %q, got %q", "foo", userInfo.Name)
}
}

58
docs/auth/token/util.go
View file

@ -1,58 +0,0 @@
package token
import (
"encoding/base64"
"errors"
"strings"
)
// joseBase64UrlEncode encodes the given data using the standard base64 url
// encoding format but with all trailing '=' characters omitted in accordance
// with the jose specification.
// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2
func joseBase64UrlEncode(b []byte) string {
return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}
// joseBase64UrlDecode decodes the given string using the standard base64 url
// decoder but first adds the appropriate number of trailing '=' characters in
// accordance with the jose specification.
// http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2
func joseBase64UrlDecode(s string) ([]byte, error) {
switch len(s) % 4 {
case 0:
case 2:
s += "=="
case 3:
s += "="
default:
return nil, errors.New("illegal base64url string")
}
return base64.URLEncoding.DecodeString(s)
}
// actionSet is a special type of stringSet.
type actionSet struct {
stringSet
}
func newActionSet(actions ...string) actionSet {
return actionSet{newStringSet(actions...)}
}
// Contains calls StringSet.Contains() for
// either "*" or the given action string.
func (s actionSet) contains(action string) bool {
return s.stringSet.contains("*") || s.stringSet.contains(action)
}
// contains returns true if q is found in ss.
func contains(ss []string, q string) bool {
for _, s := range ss {
if s == q {
return true
}
}
return false
}

58
docs/client/auth/api_version.go
View file

@ -1,58 +0,0 @@
package auth
import (
"net/http"
"strings"
)
// APIVersion represents a version of an API including its
// type and version number.
type APIVersion struct {
// Type refers to the name of a specific API specification
// such as "registry"
Type string
// Version is the version of the API specification implemented,
// This may omit the revision number and only include
// the major and minor version, such as "2.0"
Version string
}
// String returns the string formatted API Version
func (v APIVersion) String() string {
return v.Type + "/" + v.Version
}
// APIVersions gets the API versions out of an HTTP response using the provided
// version header as the key for the HTTP header.
func APIVersions(resp *http.Response, versionHeader string) []APIVersion {
versions := []APIVersion{}
if versionHeader != "" {
for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey(versionHeader)] {
for _, version := range strings.Fields(supportedVersions) {
versions = append(versions, ParseAPIVersion(version))
}
}
}
return versions
}
// ParseAPIVersion parses an API version string into an APIVersion
// Format (Expected, not enforced):
// API version string = <API type> '/' <API version>
// API type = [a-z][a-z0-9]*
// API version = [0-9]+(\.[0-9]+)?
// TODO(dmcgowan): Enforce format, add error condition, remove unknown type
func ParseAPIVersion(versionStr string) APIVersion {
idx := strings.IndexRune(versionStr, '/')
if idx == -1 {
return APIVersion{
Type: "unknown",
Version: versionStr,
}
}
return APIVersion{
Type: strings.ToLower(versionStr[:idx]),
Version: versionStr[idx+1:],
}
}

220
docs/client/auth/authchallenge.go
View file

@ -1,220 +0,0 @@
package auth
import (
"fmt"
"net/http"
"net/url"
"strings"
)
// Challenge carries information from a WWW-Authenticate response header.
// See RFC 2617.
type Challenge struct {
// Scheme is the auth-scheme according to RFC 2617
Scheme string
// Parameters are the auth-params according to RFC 2617
Parameters map[string]string
}
// ChallengeManager manages the challenges for endpoints.
// The challenges are pulled out of HTTP responses. Only
// responses which expect challenges should be added to
// the manager, since a non-unauthorized request will be
// viewed as not requiring challenges.
type ChallengeManager interface {
// GetChallenges returns the challenges for the given
// endpoint URL.
GetChallenges(endpoint url.URL) ([]Challenge, error)
// AddResponse adds the response to the challenge
// manager. The challenges will be parsed out of
// the WWW-Authenicate headers and added to the
// URL which was produced the response. If the
// response was authorized, any challenges for the
// endpoint will be cleared.
AddResponse(resp *http.Response) error
}
// NewSimpleChallengeManager returns an instance of
// ChallengeManger which only maps endpoints to challenges
// based on the responses which have been added the
// manager. The simple manager will make no attempt to
// perform requests on the endpoints or cache the responses
// to a backend.
func NewSimpleChallengeManager() ChallengeManager {
return simpleChallengeManager{}
}
type simpleChallengeManager map[string][]Challenge
func (m simpleChallengeManager) GetChallenges(endpoint url.URL) ([]Challenge, error) {
endpoint.Host = strings.ToLower(endpoint.Host)
challenges := m[endpoint.String()]
return challenges, nil
}
func (m simpleChallengeManager) AddResponse(resp *http.Response) error {
challenges := ResponseChallenges(resp)
if resp.Request == nil {
return fmt.Errorf("missing request reference")
}
urlCopy := url.URL{
Path: resp.Request.URL.Path,
Host: strings.ToLower(resp.Request.URL.Host),
Scheme: resp.Request.URL.Scheme,
}
m[urlCopy.String()] = challenges
return nil
}
// Octet types from RFC 2616.
type octetType byte
var octetTypes [256]octetType
const (
isToken octetType = 1 << iota
isSpace
)
func init() {
// OCTET = <any 8-bit sequence of data>
// CHAR = <any US-ASCII character (octets 0 - 127)>
// CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
// CR = <US-ASCII CR, carriage return (13)>
// LF = <US-ASCII LF, linefeed (10)>
// SP = <US-ASCII SP, space (32)>
// HT = <US-ASCII HT, horizontal-tab (9)>
// <"> = <US-ASCII double-quote mark (34)>
// CRLF = CR LF
// LWS = [CRLF] 1*( SP | HT )
// TEXT = <any OCTET except CTLs, but including LWS>
// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
// | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {
t |= isToken
}
octetTypes[c] = t
}
}
// ResponseChallenges returns a list of authorization challenges
// for the given http Response. Challenges are only checked if
// the response status code was a 401.
func ResponseChallenges(resp *http.Response) []Challenge {
if resp.StatusCode == http.StatusUnauthorized {
// Parse the WWW-Authenticate Header and store the challenges
// on this endpoint object.
return parseAuthHeader(resp.Header)
}
return nil
}
func parseAuthHeader(header http.Header) []Challenge {
challenges := []Challenge{}
for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] {
v, p := parseValueAndParams(h)
if v != "" {
challenges = append(challenges, Challenge{Scheme: v, Parameters: p})
}
}
return challenges
}
func parseValueAndParams(header string) (value string, params map[string]string) {
params = make(map[string]string)
value, s := expectToken(header)
if value == "" {
return
}
value = strings.ToLower(value)
s = "," + skipSpace(s)
for strings.HasPrefix(s, ",") {
var pkey string
pkey, s = expectToken(skipSpace(s[1:]))
if pkey == "" {
return
}
if !strings.HasPrefix(s, "=") {
return
}
var pvalue string
pvalue, s = expectTokenOrQuoted(s[1:])
if pvalue == "" {
return
}
pkey = strings.ToLower(pkey)
params[pkey] = pvalue
s = skipSpace(s)
}
return
}
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isSpace == 0 {
break
}
}
return s[i:]
}
func expectToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isToken == 0 {
break
}
}
return s[:i], s[i:]
}
func expectTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return expectToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + 1; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}

81
docs/client/auth/authchallenge_test.go
View file

@ -1,81 +0,0 @@
package auth
import (
"fmt"
"net/http"
"net/url"
"strings"
"testing"
)
func TestAuthChallengeParse(t *testing.T) {
header := http.Header{}
header.Add("WWW-Authenticate", `Bearer realm="https://auth.example.com/token",service="registry.example.com",other=fun,slashed="he\"\l\lo"`)
challenges := parseAuthHeader(header)
if len(challenges) != 1 {
t.Fatalf("Unexpected number of auth challenges: %d, expected 1", len(challenges))
}
challenge := challenges[0]
if expected := "bearer"; challenge.Scheme != expected {
t.Fatalf("Unexpected scheme: %s, expected: %s", challenge.Scheme, expected)
}
if expected := "https://auth.example.com/token"; challenge.Parameters["realm"] != expected {
t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["realm"], expected)
}
if expected := "registry.example.com"; challenge.Parameters["service"] != expected {
t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["service"], expected)
}
if expected := "fun"; challenge.Parameters["other"] != expected {
t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["other"], expected)
}
if expected := "he\"llo"; challenge.Parameters["slashed"] != expected {
t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["slashed"], expected)
}
}
func TestAuthChallengeNormalization(t *testing.T) {
testAuthChallengeNormalization(t, "reg.EXAMPLE.com")
testAuthChallengeNormalization(t, "bɿɒʜɔiɿ-ɿɘƚƨim-ƚol-ɒ-ƨʞnɒʜƚ.com")
}
func testAuthChallengeNormalization(t *testing.T, host string) {
scm := NewSimpleChallengeManager()
url, err := url.Parse(fmt.Sprintf("http://%s/v2/", host))
if err != nil {
t.Fatal(err)
}
resp := &http.Response{
Request: &http.Request{
URL: url,
},
Header: make(http.Header),
StatusCode: http.StatusUnauthorized,
}
resp.Header.Add("WWW-Authenticate", fmt.Sprintf("Bearer realm=\"https://%s/token\",service=\"registry.example.com\"", host))
err = scm.AddResponse(resp)
if err != nil {
t.Fatal(err)
}
lowered := *url
lowered.Host = strings.ToLower(lowered.Host)
c, err := scm.GetChallenges(lowered)
if err != nil {
t.Fatal(err)
}
if len(c) == 0 {
t.Fatal("Expected challenge for lower-cased-host URL")
}
}

480
docs/client/auth/session.go
View file

@ -1,480 +0,0 @@
package auth
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/transport"
)
var (
// ErrNoBasicAuthCredentials is returned if a request can't be authorized with
// basic auth due to lack of credentials.
ErrNoBasicAuthCredentials = errors.New("no basic auth credentials")
// ErrNoToken is returned if a request is successful but the body does not
// contain an authorization token.
ErrNoToken = errors.New("authorization server did not include a token in the response")
)
const defaultClientID = "registry-client"
// AuthenticationHandler is an interface for authorizing a request from
// params from a "WWW-Authenicate" header for a single scheme.
type AuthenticationHandler interface {
// Scheme returns the scheme as expected from the "WWW-Authenicate" header.
Scheme() string
// AuthorizeRequest adds the authorization header to a request (if needed)
// using the parameters from "WWW-Authenticate" method. The parameters
// values depend on the scheme.
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// CredentialStore is an interface for getting credentials for
// a given URL
type CredentialStore interface {
// Basic returns basic auth for the given URL
Basic(*url.URL) (string, string)
// RefreshToken returns a refresh token for the
// given URL and service
RefreshToken(*url.URL, string) string
// SetRefreshToken sets the refresh token if none
// is provided for the given url and service
SetRefreshToken(realm *url.URL, service, token string)
}
// NewAuthorizer creates an authorizer which can handle multiple authentication
// schemes. The handlers are tried in order, the higher priority authentication
// methods should be first. The challengeMap holds a list of challenges for
// a given root API endpoint (for example "https://registry-1.docker.io/v2/").
func NewAuthorizer(manager ChallengeManager, handlers ...AuthenticationHandler) transport.RequestModifier {
return &endpointAuthorizer{
challenges: manager,
handlers: handlers,
}
}
type endpointAuthorizer struct {
challenges ChallengeManager
handlers []AuthenticationHandler
transport http.RoundTripper
}
func (ea *endpointAuthorizer) ModifyRequest(req *http.Request) error {
v2Root := strings.Index(req.URL.Path, "/v2/")
if v2Root == -1 {
return nil
}
ping := url.URL{
Host: req.URL.Host,
Scheme: req.URL.Scheme,
Path: req.URL.Path[:v2Root+4],
}
challenges, err := ea.challenges.GetChallenges(ping)
if err != nil {
return err
}
if len(challenges) > 0 {
for _, handler := range ea.handlers {
for _, challenge := range challenges {
if challenge.Scheme != handler.Scheme() {
continue
}
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return err
}
}
}
}
return nil
}
// This is the minimum duration a token can last (in seconds).
// A token must not live less than 60 seconds because older versions
// of the Docker client didn't read their expiration from the token
// response and assumed 60 seconds. So to remain compatible with
// those implementations, a token must live at least this long.
const minimumTokenLifetimeSeconds = 60
// Private interface for time used by this package to enable tests to provide their own implementation.
type clock interface {
Now() time.Time
}
type tokenHandler struct {
header http.Header
creds CredentialStore
transport http.RoundTripper
clock clock
offlineAccess bool
forceOAuth bool
clientID string
scopes []Scope
tokenLock sync.Mutex
tokenCache string
tokenExpiration time.Time
}
// Scope is a type which is serializable to a string
// using the allow scope grammar.
type Scope interface {
String() string
}
// RepositoryScope represents a token scope for access
// to a repository.
type RepositoryScope struct {
Repository string
Actions []string
}
// String returns the string representation of the repository
// using the scope grammar
func (rs RepositoryScope) String() string {
return fmt.Sprintf("repository:%s:%s", rs.Repository, strings.Join(rs.Actions, ","))
}
// TokenHandlerOptions is used to configure a new token handler
type TokenHandlerOptions struct {
Transport http.RoundTripper
Credentials CredentialStore
OfflineAccess bool
ForceOAuth bool
ClientID string
Scopes []Scope
}
// An implementation of clock for providing real time data.
type realClock struct{}
// Now implements clock
func (realClock) Now() time.Time { return time.Now() }
// NewTokenHandler creates a new AuthenicationHandler which supports
// fetching tokens from a remote token server.
func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope string, actions ...string) AuthenticationHandler {
// Create options...
return NewTokenHandlerWithOptions(TokenHandlerOptions{
Transport: transport,
Credentials: creds,
Scopes: []Scope{
RepositoryScope{
Repository: scope,
Actions: actions,
},
},
})
}
// NewTokenHandlerWithOptions creates a new token handler using the provided
// options structure.
func NewTokenHandlerWithOptions(options TokenHandlerOptions) AuthenticationHandler {
handler := &tokenHandler{
transport: options.Transport,
creds: options.Credentials,
offlineAccess: options.OfflineAccess,
forceOAuth: options.ForceOAuth,
clientID: options.ClientID,
scopes: options.Scopes,
clock: realClock{},
}
return handler
}
func (th *tokenHandler) client() *http.Client {
return &http.Client{
Transport: th.transport,
Timeout: 15 * time.Second,
}
}
func (th *tokenHandler) Scheme() string {
return "bearer"
}
func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
var additionalScopes []string
if fromParam := req.URL.Query().Get("from"); fromParam != "" {
additionalScopes = append(additionalScopes, RepositoryScope{
Repository: fromParam,
Actions: []string{"pull"},
}.String())
}
token, err := th.getToken(params, additionalScopes...)
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
return nil
}
func (th *tokenHandler) getToken(params map[string]string, additionalScopes ...string) (string, error) {
th.tokenLock.Lock()
defer th.tokenLock.Unlock()
scopes := make([]string, 0, len(th.scopes)+len(additionalScopes))
for _, scope := range th.scopes {
scopes = append(scopes, scope.String())
}
var addedScopes bool
for _, scope := range additionalScopes {
scopes = append(scopes, scope)
addedScopes = true
}
now := th.clock.Now()
if now.After(th.tokenExpiration) || addedScopes {
token, expiration, err := th.fetchToken(params, scopes)
if err != nil {
return "", err
}
// do not update cache for added scope tokens
if !addedScopes {
th.tokenCache = token
th.tokenExpiration = expiration
}
return token, nil
}
return th.tokenCache, nil
}
type postTokenResponse struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
ExpiresIn int `json:"expires_in"`
IssuedAt time.Time `json:"issued_at"`
Scope string `json:"scope"`
}
func (th *tokenHandler) fetchTokenWithOAuth(realm *url.URL, refreshToken, service string, scopes []string) (token string, expiration time.Time, err error) {
form := url.Values{}
form.Set("scope", strings.Join(scopes, " "))
form.Set("service", service)
clientID := th.clientID
if clientID == "" {
// Use default client, this is a required field
clientID = defaultClientID
}
form.Set("client_id", clientID)
if refreshToken != "" {
form.Set("grant_type", "refresh_token")
form.Set("refresh_token", refreshToken)
} else if th.creds != nil {
form.Set("grant_type", "password")
username, password := th.creds.Basic(realm)
form.Set("username", username)
form.Set("password", password)
// attempt to get a refresh token
form.Set("access_type", "offline")
} else {
// refuse to do oauth without a grant type
return "", time.Time{}, fmt.Errorf("no supported grant type")
}
resp, err := th.client().PostForm(realm.String(), form)
if err != nil {
return "", time.Time{}, err
}
defer resp.Body.Close()
if !client.SuccessStatus(resp.StatusCode) {
err := client.HandleErrorResponse(resp)
return "", time.Time{}, err
}
decoder := json.NewDecoder(resp.Body)
var tr postTokenResponse
if err = decoder.Decode(&tr); err != nil {
return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err)
}
if tr.RefreshToken != "" && tr.RefreshToken != refreshToken {
th.creds.SetRefreshToken(realm, service, tr.RefreshToken)
}
if tr.ExpiresIn < minimumTokenLifetimeSeconds {
// The default/minimum lifetime.
tr.ExpiresIn = minimumTokenLifetimeSeconds
logrus.Debugf("Increasing token expiration to: %d seconds", tr.ExpiresIn)
}
if tr.IssuedAt.IsZero() {
// issued_at is optional in the token response.
tr.IssuedAt = th.clock.Now().UTC()
}
return tr.AccessToken, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil
}
type getTokenResponse struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
IssuedAt time.Time `json:"issued_at"`
RefreshToken string `json:"refresh_token"`
}
func (th *tokenHandler) fetchTokenWithBasicAuth(realm *url.URL, service string, scopes []string) (token string, expiration time.Time, err error) {
req, err := http.NewRequest("GET", realm.String(), nil)
if err != nil {
return "", time.Time{}, err
}
reqParams := req.URL.Query()
if service != "" {
reqParams.Add("service", service)
}
for _, scope := range scopes {
reqParams.Add("scope", scope)
}
if th.offlineAccess {
reqParams.Add("offline_token", "true")
clientID := th.clientID
if clientID == "" {
clientID = defaultClientID
}
reqParams.Add("client_id", clientID)
}
if th.creds != nil {
username, password := th.creds.Basic(realm)
if username != "" && password != "" {
reqParams.Add("account", username)
req.SetBasicAuth(username, password)
}
}
req.URL.RawQuery = reqParams.Encode()
resp, err := th.client().Do(req)
if err != nil {
return "", time.Time{}, err
}
defer resp.Body.Close()
if !client.SuccessStatus(resp.StatusCode) {
err := client.HandleErrorResponse(resp)
return "", time.Time{}, err
}
decoder := json.NewDecoder(resp.Body)
var tr getTokenResponse
if err = decoder.Decode(&tr); err != nil {
return "", time.Time{}, fmt.Errorf("unable to decode token response: %s", err)
}
if tr.RefreshToken != "" && th.creds != nil {
th.creds.SetRefreshToken(realm, service, tr.RefreshToken)
}
// `access_token` is equivalent to `token` and if both are specified
// the choice is undefined. Canonicalize `access_token` by sticking
// things in `token`.
if tr.AccessToken != "" {
tr.Token = tr.AccessToken
}
if tr.Token == "" {
return "", time.Time{}, ErrNoToken
}
if tr.ExpiresIn < minimumTokenLifetimeSeconds {
// The default/minimum lifetime.
tr.ExpiresIn = minimumTokenLifetimeSeconds
logrus.Debugf("Increasing token expiration to: %d seconds", tr.ExpiresIn)
}
if tr.IssuedAt.IsZero() {
// issued_at is optional in the token response.
tr.IssuedAt = th.clock.Now().UTC()
}
return tr.Token, tr.IssuedAt.Add(time.Duration(tr.ExpiresIn) * time.Second), nil
}
func (th *tokenHandler) fetchToken(params map[string]string, scopes []string) (token string, expiration time.Time, err error) {
realm, ok := params["realm"]
if !ok {
return "", time.Time{}, errors.New("no realm specified for token auth challenge")
}
// TODO(dmcgowan): Handle empty scheme and relative realm
realmURL, err := url.Parse(realm)
if err != nil {
return "", time.Time{}, fmt.Errorf("invalid token auth challenge realm: %s", err)
}
service := params["service"]
var refreshToken string
if th.creds != nil {
refreshToken = th.creds.RefreshToken(realmURL, service)
}
if refreshToken != "" || th.forceOAuth {
return th.fetchTokenWithOAuth(realmURL, refreshToken, service, scopes)
}
return th.fetchTokenWithBasicAuth(realmURL, service, scopes)
}
type basicHandler struct {
creds CredentialStore
}
// NewBasicHandler creaters a new authentiation handler which adds
// basic authentication credentials to a request.
func NewBasicHandler(creds CredentialStore) AuthenticationHandler {
return &basicHandler{
creds: creds,
}
}
func (*basicHandler) Scheme() string {
return "basic"
}
func (bh *basicHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
if bh.creds != nil {
username, password := bh.creds.Basic(req.URL)
if username != "" && password != "" {
req.SetBasicAuth(username, password)
return nil
}
}
return ErrNoBasicAuthCredentials
}

787
docs/client/auth/session_test.go
View file

@ -1,787 +0,0 @@
package auth
import (
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/distribution/testutil"
)
// An implementation of clock for providing fake time data.
type fakeClock struct {
current time.Time
}
// Now implements clock
func (fc *fakeClock) Now() time.Time { return fc.current }
func testServer(rrm testutil.RequestResponseMap) (string, func()) {
h := testutil.NewHandler(rrm)
s := httptest.NewServer(h)
return s.URL, s.Close
}
type testAuthenticationWrapper struct {
headers http.Header
authCheck func(string) bool
next http.Handler
}
func (w *testAuthenticationWrapper) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
auth := r.Header.Get("Authorization")
if auth == "" || !w.authCheck(auth) {
h := rw.Header()
for k, values := range w.headers {
h[k] = values
}
rw.WriteHeader(http.StatusUnauthorized)
return
}
w.next.ServeHTTP(rw, r)
}
func testServerWithAuth(rrm testutil.RequestResponseMap, authenticate string, authCheck func(string) bool) (string, func()) {
h := testutil.NewHandler(rrm)
wrapper := &testAuthenticationWrapper{
headers: http.Header(map[string][]string{
"X-API-Version": {"registry/2.0"},
"X-Multi-API-Version": {"registry/2.0", "registry/2.1", "trust/1.0"},
"WWW-Authenticate": {authenticate},
}),
authCheck: authCheck,
next: h,
}
s := httptest.NewServer(wrapper)
return s.URL, s.Close
}
// ping pings the provided endpoint to determine its required authorization challenges.
// If a version header is provided, the versions will be returned.
func ping(manager ChallengeManager, endpoint, versionHeader string) ([]APIVersion, error) {
resp, err := http.Get(endpoint)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := manager.AddResponse(resp); err != nil {
return nil, err
}
return APIVersions(resp, versionHeader), err
}
type testCredentialStore struct {
username string
password string
refreshTokens map[string]string
}
func (tcs *testCredentialStore) Basic(*url.URL) (string, string) {
return tcs.username, tcs.password
}
func (tcs *testCredentialStore) RefreshToken(u *url.URL, service string) string {
return tcs.refreshTokens[service]
}
func (tcs *testCredentialStore) SetRefreshToken(u *url.URL, service string, token string) {
if tcs.refreshTokens != nil {
tcs.refreshTokens[service] = token
}
}
func TestEndpointAuthorizeToken(t *testing.T) {
service := "localhost.localdomain"
repo1 := "some/registry"
repo2 := "other/registry"
scope1 := fmt.Sprintf("repository:%s:pull,push", repo1)
scope2 := fmt.Sprintf("repository:%s:pull,push", repo2)
tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?scope=%s&service=%s", url.QueryEscape(scope1), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"token":"statictoken"}`),
},
},
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?scope=%s&service=%s", url.QueryEscape(scope2), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"token":"badtoken"}`),
},
},
})
te, tc := testServer(tokenMap)
defer tc()
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
authenicate := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service)
validCheck := func(a string) bool {
return a == "Bearer statictoken"
}
e, c := testServerWithAuth(m, authenicate, validCheck)
defer c()
challengeManager1 := NewSimpleChallengeManager()
versions, err := ping(challengeManager1, e+"/v2/", "x-api-version")
if err != nil {
t.Fatal(err)
}
if len(versions) != 1 {
t.Fatalf("Unexpected version count: %d, expected 1", len(versions))
}
if check := (APIVersion{Type: "registry", Version: "2.0"}); versions[0] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[0], check)
}
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager1, NewTokenHandler(nil, nil, repo1, "pull", "push")))
client := &http.Client{Transport: transport1}
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
e2, c2 := testServerWithAuth(m, authenicate, validCheck)
defer c2()
challengeManager2 := NewSimpleChallengeManager()
versions, err = ping(challengeManager2, e2+"/v2/", "x-multi-api-version")
if err != nil {
t.Fatal(err)
}
if len(versions) != 3 {
t.Fatalf("Unexpected version count: %d, expected 3", len(versions))
}
if check := (APIVersion{Type: "registry", Version: "2.0"}); versions[0] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[0], check)
}
if check := (APIVersion{Type: "registry", Version: "2.1"}); versions[1] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[1], check)
}
if check := (APIVersion{Type: "trust", Version: "1.0"}); versions[2] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[2], check)
}
transport2 := transport.NewTransport(nil, NewAuthorizer(challengeManager2, NewTokenHandler(nil, nil, repo2, "pull", "push")))
client2 := &http.Client{Transport: transport2}
req, _ = http.NewRequest("GET", e2+"/v2/hello", nil)
resp, err = client2.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusUnauthorized)
}
}
func TestEndpointAuthorizeRefreshToken(t *testing.T) {
service := "localhost.localdomain"
repo1 := "some/registry"
repo2 := "other/registry"
scope1 := fmt.Sprintf("repository:%s:pull,push", repo1)
scope2 := fmt.Sprintf("repository:%s:pull,push", repo2)
refreshToken1 := "0123456790abcdef"
refreshToken2 := "0123456790fedcba"
tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "POST",
Route: "/token",
Body: []byte(fmt.Sprintf("client_id=registry-client&grant_type=refresh_token&refresh_token=%s&scope=%s&service=%s", refreshToken1, url.QueryEscape(scope1), service)),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(fmt.Sprintf(`{"access_token":"statictoken","refresh_token":"%s"}`, refreshToken1)),
},
},
{
// In the future this test may fail and require using basic auth to get a different refresh token
Request: testutil.Request{
Method: "POST",
Route: "/token",
Body: []byte(fmt.Sprintf("client_id=registry-client&grant_type=refresh_token&refresh_token=%s&scope=%s&service=%s", refreshToken1, url.QueryEscape(scope2), service)),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(fmt.Sprintf(`{"access_token":"statictoken","refresh_token":"%s"}`, refreshToken2)),
},
},
{
Request: testutil.Request{
Method: "POST",
Route: "/token",
Body: []byte(fmt.Sprintf("client_id=registry-client&grant_type=refresh_token&refresh_token=%s&scope=%s&service=%s", refreshToken2, url.QueryEscape(scope2), service)),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"access_token":"badtoken","refresh_token":"%s"}`),
},
},
})
te, tc := testServer(tokenMap)
defer tc()
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
authenicate := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service)
validCheck := func(a string) bool {
return a == "Bearer statictoken"
}
e, c := testServerWithAuth(m, authenicate, validCheck)
defer c()
challengeManager1 := NewSimpleChallengeManager()
versions, err := ping(challengeManager1, e+"/v2/", "x-api-version")
if err != nil {
t.Fatal(err)
}
if len(versions) != 1 {
t.Fatalf("Unexpected version count: %d, expected 1", len(versions))
}
if check := (APIVersion{Type: "registry", Version: "2.0"}); versions[0] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[0], check)
}
creds := &testCredentialStore{
refreshTokens: map[string]string{
service: refreshToken1,
},
}
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager1, NewTokenHandler(nil, creds, repo1, "pull", "push")))
client := &http.Client{Transport: transport1}
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
// Try with refresh token setting
e2, c2 := testServerWithAuth(m, authenicate, validCheck)
defer c2()
challengeManager2 := NewSimpleChallengeManager()
versions, err = ping(challengeManager2, e2+"/v2/", "x-api-version")
if err != nil {
t.Fatal(err)
}
if len(versions) != 1 {
t.Fatalf("Unexpected version count: %d, expected 1", len(versions))
}
if check := (APIVersion{Type: "registry", Version: "2.0"}); versions[0] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[0], check)
}
transport2 := transport.NewTransport(nil, NewAuthorizer(challengeManager2, NewTokenHandler(nil, creds, repo2, "pull", "push")))
client2 := &http.Client{Transport: transport2}
req, _ = http.NewRequest("GET", e2+"/v2/hello", nil)
resp, err = client2.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusUnauthorized)
}
if creds.refreshTokens[service] != refreshToken2 {
t.Fatalf("Refresh token not set after change")
}
// Try with bad token
e3, c3 := testServerWithAuth(m, authenicate, validCheck)
defer c3()
challengeManager3 := NewSimpleChallengeManager()
versions, err = ping(challengeManager3, e3+"/v2/", "x-api-version")
if err != nil {
t.Fatal(err)
}
if check := (APIVersion{Type: "registry", Version: "2.0"}); versions[0] != check {
t.Fatalf("Unexpected api version: %#v, expected %#v", versions[0], check)
}
transport3 := transport.NewTransport(nil, NewAuthorizer(challengeManager3, NewTokenHandler(nil, creds, repo2, "pull", "push")))
client3 := &http.Client{Transport: transport3}
req, _ = http.NewRequest("GET", e3+"/v2/hello", nil)
resp, err = client3.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusUnauthorized)
}
}
func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}
func TestEndpointAuthorizeTokenBasic(t *testing.T) {
service := "localhost.localdomain"
repo := "some/fun/registry"
scope := fmt.Sprintf("repository:%s:pull,push", repo)
username := "tokenuser"
password := "superSecretPa$$word"
tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"access_token":"statictoken"}`),
},
},
})
authenicate1 := fmt.Sprintf("Basic realm=localhost")
basicCheck := func(a string) bool {
return a == fmt.Sprintf("Basic %s", basicAuth(username, password))
}
te, tc := testServerWithAuth(tokenMap, authenicate1, basicCheck)
defer tc()
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service)
bearerCheck := func(a string) bool {
return a == "Bearer statictoken"
}
e, c := testServerWithAuth(m, authenicate2, bearerCheck)
defer c()
creds := &testCredentialStore{
username: username,
password: password,
}
challengeManager := NewSimpleChallengeManager()
_, err := ping(challengeManager, e+"/v2/", "")
if err != nil {
t.Fatal(err)
}
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager, NewTokenHandler(nil, creds, repo, "pull", "push"), NewBasicHandler(creds)))
client := &http.Client{Transport: transport1}
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
}
func TestEndpointAuthorizeTokenBasicWithExpiresIn(t *testing.T) {
service := "localhost.localdomain"
repo := "some/fun/registry"
scope := fmt.Sprintf("repository:%s:pull,push", repo)
username := "tokenuser"
password := "superSecretPa$$word"
tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"token":"statictoken", "expires_in": 3001}`),
},
},
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"access_token":"statictoken", "expires_in": 3001}`),
},
},
})
authenicate1 := fmt.Sprintf("Basic realm=localhost")
tokenExchanges := 0
basicCheck := func(a string) bool {
tokenExchanges = tokenExchanges + 1
return a == fmt.Sprintf("Basic %s", basicAuth(username, password))
}
te, tc := testServerWithAuth(tokenMap, authenicate1, basicCheck)
defer tc()
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service)
bearerCheck := func(a string) bool {
return a == "Bearer statictoken"
}
e, c := testServerWithAuth(m, authenicate2, bearerCheck)
defer c()
creds := &testCredentialStore{
username: username,
password: password,
}
challengeManager := NewSimpleChallengeManager()
_, err := ping(challengeManager, e+"/v2/", "")
if err != nil {
t.Fatal(err)
}
clock := &fakeClock{current: time.Now()}
options := TokenHandlerOptions{
Transport: nil,
Credentials: creds,
Scopes: []Scope{
RepositoryScope{
Repository: repo,
Actions: []string{"pull", "push"},
},
},
}
tHandler := NewTokenHandlerWithOptions(options)
tHandler.(*tokenHandler).clock = clock
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager, tHandler, NewBasicHandler(creds)))
client := &http.Client{Transport: transport1}
// First call should result in a token exchange
// Subsequent calls should recycle the token from the first request, until the expiration has lapsed.
timeIncrement := 1000 * time.Second
for i := 0; i < 4; i++ {
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
if tokenExchanges != 1 {
t.Fatalf("Unexpected number of token exchanges, want: 1, got %d (iteration: %d)", tokenExchanges, i)
}
clock.current = clock.current.Add(timeIncrement)
}
// After we've exceeded the expiration, we should see a second token exchange.
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
if tokenExchanges != 2 {
t.Fatalf("Unexpected number of token exchanges, want: 2, got %d", tokenExchanges)
}
}
func TestEndpointAuthorizeTokenBasicWithExpiresInAndIssuedAt(t *testing.T) {
service := "localhost.localdomain"
repo := "some/fun/registry"
scope := fmt.Sprintf("repository:%s:pull,push", repo)
username := "tokenuser"
password := "superSecretPa$$word"
// This test sets things up such that the token was issued one increment
// earlier than its sibling in TestEndpointAuthorizeTokenBasicWithExpiresIn.
// This will mean that the token expires after 3 increments instead of 4.
clock := &fakeClock{current: time.Now()}
timeIncrement := 1000 * time.Second
firstIssuedAt := clock.Now()
clock.current = clock.current.Add(timeIncrement)
secondIssuedAt := clock.current.Add(2 * timeIncrement)
tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"token":"statictoken", "issued_at": "` + firstIssuedAt.Format(time.RFC3339Nano) + `", "expires_in": 3001}`),
},
},
{
Request: testutil.Request{
Method: "GET",
Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service),
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: []byte(`{"access_token":"statictoken", "issued_at": "` + secondIssuedAt.Format(time.RFC3339Nano) + `", "expires_in": 3001}`),
},
},
})
authenicate1 := fmt.Sprintf("Basic realm=localhost")
tokenExchanges := 0
basicCheck := func(a string) bool {
tokenExchanges = tokenExchanges + 1
return a == fmt.Sprintf("Basic %s", basicAuth(username, password))
}
te, tc := testServerWithAuth(tokenMap, authenicate1, basicCheck)
defer tc()
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service)
bearerCheck := func(a string) bool {
return a == "Bearer statictoken"
}
e, c := testServerWithAuth(m, authenicate2, bearerCheck)
defer c()
creds := &testCredentialStore{
username: username,
password: password,
}
challengeManager := NewSimpleChallengeManager()
_, err := ping(challengeManager, e+"/v2/", "")
if err != nil {
t.Fatal(err)
}
options := TokenHandlerOptions{
Transport: nil,
Credentials: creds,
Scopes: []Scope{
RepositoryScope{
Repository: repo,
Actions: []string{"pull", "push"},
},
},
}
tHandler := NewTokenHandlerWithOptions(options)
tHandler.(*tokenHandler).clock = clock
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager, tHandler, NewBasicHandler(creds)))
client := &http.Client{Transport: transport1}
// First call should result in a token exchange
// Subsequent calls should recycle the token from the first request, until the expiration has lapsed.
// We shaved one increment off of the equivalent logic in TestEndpointAuthorizeTokenBasicWithExpiresIn
// so this loop should have one fewer iteration.
for i := 0; i < 3; i++ {
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
if tokenExchanges != 1 {
t.Fatalf("Unexpected number of token exchanges, want: 1, got %d (iteration: %d)", tokenExchanges, i)
}
clock.current = clock.current.Add(timeIncrement)
}
// After we've exceeded the expiration, we should see a second token exchange.
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
if tokenExchanges != 2 {
t.Fatalf("Unexpected number of token exchanges, want: 2, got %d", tokenExchanges)
}
}
func TestEndpointAuthorizeBasic(t *testing.T) {
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/hello",
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
},
},
})
username := "user1"
password := "funSecretPa$$word"
authenicate := fmt.Sprintf("Basic realm=localhost")
validCheck := func(a string) bool {
return a == fmt.Sprintf("Basic %s", basicAuth(username, password))
}
e, c := testServerWithAuth(m, authenicate, validCheck)
defer c()
creds := &testCredentialStore{
username: username,
password: password,
}
challengeManager := NewSimpleChallengeManager()
_, err := ping(challengeManager, e+"/v2/", "")
if err != nil {
t.Fatal(err)
}
transport1 := transport.NewTransport(nil, NewAuthorizer(challengeManager, NewBasicHandler(creds)))
client := &http.Client{Transport: transport1}
req, _ := http.NewRequest("GET", e+"/v2/hello", nil)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Error sending get request: %s", err)
}
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted)
}
}

162
docs/client/blob_writer.go
View file

@ -1,162 +0,0 @@
package client
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
type httpBlobUpload struct {
statter distribution.BlobStatter
client *http.Client
uuid string
startedAt time.Time
location string // always the last value of the location header.
offset int64
closed bool
}
func (hbu *httpBlobUpload) Reader() (io.ReadCloser, error) {
panic("Not implemented")
}
func (hbu *httpBlobUpload) handleErrorResponse(resp *http.Response) error {
if resp.StatusCode == http.StatusNotFound {
return distribution.ErrBlobUploadUnknown
}
return HandleErrorResponse(resp)
}
func (hbu *httpBlobUpload) ReadFrom(r io.Reader) (n int64, err error) {
req, err := http.NewRequest("PATCH", hbu.location, ioutil.NopCloser(r))
if err != nil {
return 0, err
}
defer req.Body.Close()
resp, err := hbu.client.Do(req)
if err != nil {
return 0, err
}
if !SuccessStatus(resp.StatusCode) {
return 0, hbu.handleErrorResponse(resp)
}
hbu.uuid = resp.Header.Get("Docker-Upload-UUID")
hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location)
if err != nil {
return 0, err
}
rng := resp.Header.Get("Range")
var start, end int64
if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil {
return 0, err
} else if n != 2 || end < start {
return 0, fmt.Errorf("bad range format: %s", rng)
}
return (end - start + 1), nil
}
func (hbu *httpBlobUpload) Write(p []byte) (n int, err error) {
req, err := http.NewRequest("PATCH", hbu.location, bytes.NewReader(p))
if err != nil {
return 0, err
}
req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hbu.offset, hbu.offset+int64(len(p)-1)))
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p)))
req.Header.Set("Content-Type", "application/octet-stream")
resp, err := hbu.client.Do(req)
if err != nil {
return 0, err
}
if !SuccessStatus(resp.StatusCode) {
return 0, hbu.handleErrorResponse(resp)
}
hbu.uuid = resp.Header.Get("Docker-Upload-UUID")
hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location)
if err != nil {
return 0, err
}
rng := resp.Header.Get("Range")
var start, end int
if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil {
return 0, err
} else if n != 2 || end < start {
return 0, fmt.Errorf("bad range format: %s", rng)
}
return (end - start + 1), nil
}
func (hbu *httpBlobUpload) Size() int64 {
return hbu.offset
}
func (hbu *httpBlobUpload) ID() string {
return hbu.uuid
}
func (hbu *httpBlobUpload) StartedAt() time.Time {
return hbu.startedAt
}
func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) {
// TODO(dmcgowan): Check if already finished, if so just fetch
req, err := http.NewRequest("PUT", hbu.location, nil)
if err != nil {
return distribution.Descriptor{}, err
}
values := req.URL.Query()
values.Set("digest", desc.Digest.String())
req.URL.RawQuery = values.Encode()
resp, err := hbu.client.Do(req)
if err != nil {
return distribution.Descriptor{}, err
}
defer resp.Body.Close()
if !SuccessStatus(resp.StatusCode) {
return distribution.Descriptor{}, hbu.handleErrorResponse(resp)
}
return hbu.statter.Stat(ctx, desc.Digest)
}
func (hbu *httpBlobUpload) Cancel(ctx context.Context) error {
req, err := http.NewRequest("DELETE", hbu.location, nil)
if err != nil {
return err
}
resp, err := hbu.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || SuccessStatus(resp.StatusCode) {
return nil
}
return hbu.handleErrorResponse(resp)
}
func (hbu *httpBlobUpload) Close() error {
hbu.closed = true
return nil
}

211
docs/client/blob_writer_test.go
View file

@ -1,211 +0,0 @@
package client
import (
"bytes"
"fmt"
"net/http"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/testutil"
)
// Test implements distribution.BlobWriter
var _ distribution.BlobWriter = &httpBlobUpload{}
func TestUploadReadFrom(t *testing.T) {
_, b := newRandomBlob(64)
repo := "test/upload/readfrom"
locationPath := fmt.Sprintf("/v2/%s/uploads/testid", repo)
m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{
{
Request: testutil.Request{
Method: "GET",
Route: "/v2/",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Headers: http.Header(map[string][]string{
"Docker-Distribution-API-Version": {"registry/2.0"},
}),
},
},
// Test Valid case
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
Headers: http.Header(map[string][]string{
"Docker-Upload-UUID": {"46603072-7a1b-4b41-98f9-fd8a7da89f9b"},
"Location": {locationPath},
"Range": {"0-63"},
}),
},
},
// Test invalid range
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
Headers: http.Header(map[string][]string{
"Docker-Upload-UUID": {"46603072-7a1b-4b41-98f9-fd8a7da89f9b"},
"Location": {locationPath},
"Range": {""},
}),
},
},
// Test 404
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusNotFound,
},
},
// Test 400 valid json
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusBadRequest,
Body: []byte(`
{ "errors":
[
{
"code": "BLOB_UPLOAD_INVALID",
"message": "blob upload invalid",
"detail": "more detail"
}
]
} `),
},
},
// Test 400 invalid json
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusBadRequest,
Body: []byte("something bad happened"),
},
},
// Test 500
{
Request: testutil.Request{
Method: "PATCH",
Route: locationPath,
Body: b,
},
Response: testutil.Response{
StatusCode: http.StatusInternalServerError,
},
},
})
e, c := testServer(m)
defer c()
blobUpload := &httpBlobUpload{
client: &http.Client{},
}
// Valid case
blobUpload.location = e + locationPath
n, err := blobUpload.ReadFrom(bytes.NewReader(b))
if err != nil {
t.Fatalf("Error calling ReadFrom: %s", err)
}
if n != 64 {
t.Fatalf("Wrong length returned from ReadFrom: %d, expected 64", n)
}
// Bad range
blobUpload.location = e + locationPath
_, err = blobUpload.ReadFrom(bytes.NewReader(b))
if err == nil {
t.Fatalf("Expected error when bad range received")
}
// 404
blobUpload.location = e + locationPath
_, err = blobUpload.ReadFrom(bytes.NewReader(b))
if err == nil {
t.Fatalf("Expected error when not found")
}
if err != distribution.ErrBlobUploadUnknown {
t.Fatalf("Wrong error thrown: %s, expected %s", err, distribution.ErrBlobUploadUnknown)
}
// 400 valid json
blobUpload.location = e + locationPath
_, err = blobUpload.ReadFrom(bytes.NewReader(b))
if err == nil {
t.Fatalf("Expected error when not found")
}
if uploadErr, ok := err.(errcode.Errors); !ok {
t.Fatalf("Wrong error type %T: %s", err, err)
} else if len(uploadErr) != 1 {
t.Fatalf("Unexpected number of errors: %d, expected 1", len(uploadErr))
} else {
v2Err, ok := uploadErr[0].(errcode.Error)
if !ok {
t.Fatalf("Not an 'Error' type: %#v", uploadErr[0])
}
if v2Err.Code != v2.ErrorCodeBlobUploadInvalid {
t.Fatalf("Unexpected error code: %s, expected %d", v2Err.Code.String(), v2.ErrorCodeBlobUploadInvalid)
}
if expected := "blob upload invalid"; v2Err.Message != expected {
t.Fatalf("Unexpected error message: %q, expected %q", v2Err.Message, expected)
}
if expected := "more detail"; v2Err.Detail.(string) != expected {
t.Fatalf("Unexpected error message: %q, expected %q", v2Err.Detail.(string), expected)
}
}
// 400 invalid json
blobUpload.location = e + locationPath
_, err = blobUpload.ReadFrom(bytes.NewReader(b))
if err == nil {
t.Fatalf("Expected error when not found")
}
if uploadErr, ok := err.(*UnexpectedHTTPResponseError); !ok {
t.Fatalf("Wrong error type %T: %s", err, err)
} else {
respStr := string(uploadErr.Response)
if expected := "something bad happened"; respStr != expected {
t.Fatalf("Unexpected response string: %s, expected: %s", respStr, expected)
}
}
// 500
blobUpload.location = e + locationPath
_, err = blobUpload.ReadFrom(bytes.NewReader(b))
if err == nil {
t.Fatalf("Expected error when not found")
}
if uploadErr, ok := err.(*UnexpectedHTTPStatusError); !ok {
t.Fatalf("Wrong error type %T: %s", err, err)
} else if expected := "500 " + http.StatusText(http.StatusInternalServerError); uploadErr.Status != expected {
t.Fatalf("Unexpected response status: %s, expected %s", uploadErr.Status, expected)
}
}

107
docs/client/errors.go
View file

@ -1,107 +0,0 @@
package client
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"github.com/docker/distribution/registry/api/errcode"
)
// ErrNoErrorsInBody is returned when an HTTP response body parses to an empty
// errcode.Errors slice.
var ErrNoErrorsInBody = errors.New("no error details found in HTTP response body")
// UnexpectedHTTPStatusError is returned when an unexpected HTTP status is
// returned when making a registry api call.
type UnexpectedHTTPStatusError struct {
Status string
}
func (e *UnexpectedHTTPStatusError) Error() string {
return fmt.Sprintf("received unexpected HTTP status: %s", e.Status)
}
// UnexpectedHTTPResponseError is returned when an expected HTTP status code
// is returned, but the content was unexpected and failed to be parsed.
type UnexpectedHTTPResponseError struct {
ParseErr error
StatusCode int
Response []byte
}
func (e *UnexpectedHTTPResponseError) Error() string {
return fmt.Sprintf("error parsing HTTP %d response body: %s: %q", e.StatusCode, e.ParseErr.Error(), string(e.Response))
}
func parseHTTPErrorResponse(statusCode int, r io.Reader) error {
var errors errcode.Errors
body, err := ioutil.ReadAll(r)
if err != nil {
return err
}
// For backward compatibility, handle irregularly formatted
// messages that contain a "details" field.
var detailsErr struct {
Details string `json:"details"`
}
err = json.Unmarshal(body, &detailsErr)
if err == nil && detailsErr.Details != "" {
switch statusCode {
case http.StatusUnauthorized:
return errcode.ErrorCodeUnauthorized.WithMessage(detailsErr.Details)
case http.StatusTooManyRequests:
return errcode.ErrorCodeTooManyRequests.WithMessage(detailsErr.Details)
default:
return errcode.ErrorCodeUnknown.WithMessage(detailsErr.Details)
}
}
if err := json.Unmarshal(body, &errors); err != nil {
return &UnexpectedHTTPResponseError{
ParseErr: err,
StatusCode: statusCode,
Response: body,
}
}
if len(errors) == 0 {
// If there was no error specified in the body, return
// UnexpectedHTTPResponseError.
return &UnexpectedHTTPResponseError{
ParseErr: ErrNoErrorsInBody,
StatusCode: statusCode,
Response: body,
}
}
return errors
}
// HandleErrorResponse returns error parsed from HTTP response for an
// unsuccessful HTTP response code (in the range 400 - 499 inclusive). An
// UnexpectedHTTPStatusError returned for response code outside of expected
// range.
func HandleErrorResponse(resp *http.Response) error {
if resp.StatusCode == 401 {
err := parseHTTPErrorResponse(resp.StatusCode, resp.Body)
if uErr, ok := err.(*UnexpectedHTTPResponseError); ok {
return errcode.ErrorCodeUnauthorized.WithDetail(uErr.Response)
}
return err
}
if resp.StatusCode >= 400 && resp.StatusCode < 500 {
return parseHTTPErrorResponse(resp.StatusCode, resp.Body)
}
return &UnexpectedHTTPStatusError{Status: resp.Status}
}
// SuccessStatus returns true if the argument is a successful HTTP response
// code (in the range 200 - 399 inclusive).
func SuccessStatus(status int) bool {
return status >= 200 && status <= 399
}

104
docs/client/errors_test.go
View file

@ -1,104 +0,0 @@
package client
import (
"bytes"
"io"
"net/http"
"strings"
"testing"
)
type nopCloser struct {
io.Reader
}
func (nopCloser) Close() error { return nil }
func TestHandleErrorResponse401ValidBody(t *testing.T) {
json := "{\"errors\":[{\"code\":\"UNAUTHORIZED\",\"message\":\"action requires authentication\"}]}"
response := &http.Response{
Status: "401 Unauthorized",
StatusCode: 401,
Body: nopCloser{bytes.NewBufferString(json)},
}
err := HandleErrorResponse(response)
expectedMsg := "unauthorized: action requires authentication"
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}
func TestHandleErrorResponse401WithInvalidBody(t *testing.T) {
json := "{invalid json}"
response := &http.Response{
Status: "401 Unauthorized",
StatusCode: 401,
Body: nopCloser{bytes.NewBufferString(json)},
}
err := HandleErrorResponse(response)
expectedMsg := "unauthorized: authentication required"
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}
func TestHandleErrorResponseExpectedStatusCode400ValidBody(t *testing.T) {
json := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest does not match\"}]}"
response := &http.Response{
Status: "400 Bad Request",
StatusCode: 400,
Body: nopCloser{bytes.NewBufferString(json)},
}
err := HandleErrorResponse(response)
expectedMsg := "digest invalid: provided digest does not match"
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}
func TestHandleErrorResponseExpectedStatusCode404EmptyErrorSlice(t *testing.T) {
json := `{"randomkey": "randomvalue"}`
response := &http.Response{
Status: "404 Not Found",
StatusCode: 404,
Body: nopCloser{bytes.NewBufferString(json)},
}
err := HandleErrorResponse(response)
expectedMsg := `error parsing HTTP 404 response body: no error details found in HTTP response body: "{\"randomkey\": \"randomvalue\"}"`
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}
func TestHandleErrorResponseExpectedStatusCode404InvalidBody(t *testing.T) {
json := "{invalid json}"
response := &http.Response{
Status: "404 Not Found",
StatusCode: 404,
Body: nopCloser{bytes.NewBufferString(json)},
}
err := HandleErrorResponse(response)
expectedMsg := "error parsing HTTP 404 response body: invalid character 'i' looking for beginning of object key string: \"{invalid json}\""
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}
func TestHandleErrorResponseUnexpectedStatusCode501(t *testing.T) {
response := &http.Response{
Status: "501 Not Implemented",
StatusCode: 501,
Body: nopCloser{bytes.NewBufferString("{\"Error Encountered\" : \"Function not implemented.\"}")},
}
err := HandleErrorResponse(response)
expectedMsg := "received unexpected HTTP status: 501 Not Implemented"
if !strings.Contains(err.Error(), expectedMsg) {
t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error())
}
}

863
docs/client/repository.go
View file

@ -1,863 +0,0 @@
package client
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/distribution/registry/storage/cache"
"github.com/docker/distribution/registry/storage/cache/memory"
)
// Registry provides an interface for calling Repositories, which returns a catalog of repositories.
type Registry interface {
Repositories(ctx context.Context, repos []string, last string) (n int, err error)
}
// checkHTTPRedirect is a callback that can manipulate redirected HTTP
// requests. It is used to preserve Accept and Range headers.
func checkHTTPRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
if len(via) > 0 {
for headerName, headerVals := range via[0].Header {
if headerName != "Accept" && headerName != "Range" {
continue
}
for _, val := range headerVals {
// Don't add to redirected request if redirected
// request already has a header with the same
// name and value.
hasValue := false
for _, existingVal := range req.Header[headerName] {
if existingVal == val {
hasValue = true
break
}
}
if !hasValue {
req.Header.Add(headerName, val)
}
}
}
}
return nil
}
// NewRegistry creates a registry namespace which can be used to get a listing of repositories
func NewRegistry(ctx context.Context, baseURL string, transport http.RoundTripper) (Registry, error) {
ub, err := v2.NewURLBuilderFromString(baseURL, false)
if err != nil {
return nil, err
}
client := &http.Client{
Transport: transport,
Timeout: 1 * time.Minute,
CheckRedirect: checkHTTPRedirect,
}
return &registry{
client: client,
ub: ub,
context: ctx,
}, nil
}
type registry struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
}
// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size
// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there
// are no more entries
func (r *registry) Repositories(ctx context.Context, entries []string, last string) (int, error) {
var numFilled int
var returnErr error
values := buildCatalogValues(len(entries), last)
u, err := r.ub.BuildCatalogURL(values)
if err != nil {
return 0, err
}
resp, err := r.client.Get(u)
if err != nil {
return 0, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(&ctlg); err != nil {
return 0, err
}
for cnt := range ctlg.Repositories {
entries[cnt] = ctlg.Repositories[cnt]
}
numFilled = len(ctlg.Repositories)
link := resp.Header.Get("Link")
if link == "" {
returnErr = io.EOF
}
} else {
return 0, HandleErrorResponse(resp)
}
return numFilled, returnErr
}
// NewRepository creates a new Repository for the given repository name and base URL.
func NewRepository(ctx context.Context, name reference.Named, baseURL string, transport http.RoundTripper) (distribution.Repository, error) {
ub, err := v2.NewURLBuilderFromString(baseURL, false)
if err != nil {
return nil, err
}
client := &http.Client{
Transport: transport,
CheckRedirect: checkHTTPRedirect,
// TODO(dmcgowan): create cookie jar
}
return &repository{
client: client,
ub: ub,
name: name,
context: ctx,
}, nil
}
type repository struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name reference.Named
}
func (r *repository) Named() reference.Named {
return r.name
}
func (r *repository) Blobs(ctx context.Context) distribution.BlobStore {
statter := &blobStatter{
name: r.name,
ub: r.ub,
client: r.client,
}
return &blobs{
name: r.name,
ub: r.ub,
client: r.client,
statter: cache.NewCachedBlobStatter(memory.NewInMemoryBlobDescriptorCacheProvider(), statter),
}
}
func (r *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
// todo(richardscothern): options should be sent over the wire
return &manifests{
name: r.name,
ub: r.ub,
client: r.client,
etags: make(map[string]string),
}, nil
}
func (r *repository) Tags(ctx context.Context) distribution.TagService {
return &tags{
client: r.client,
ub: r.ub,
context: r.context,
name: r.Named(),
}
}
// tags implements remote tagging operations.
type tags struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name reference.Named
}
// All returns all tags
func (t *tags) All(ctx context.Context) ([]string, error) {
var tags []string
u, err := t.ub.BuildTagsURL(t.name)
if err != nil {
return tags, err
}
for {
resp, err := t.client.Get(u)
if err != nil {
return tags, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return tags, err
}
tags = append(tags, tagsResponse.Tags...)
if link := resp.Header.Get("Link"); link != "" {
u = strings.Trim(strings.Split(link, ";")[0], "<>")
} else {
return tags, nil
}
} else {
return tags, HandleErrorResponse(resp)
}
}
}
func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) {
desc := distribution.Descriptor{}
headers := response.Header
ctHeader := headers.Get("Content-Type")
if ctHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Type header")
}
desc.MediaType = ctHeader
digestHeader := headers.Get("Docker-Content-Digest")
if digestHeader == "" {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return distribution.Descriptor{}, err
}
_, desc, err := distribution.UnmarshalManifest(ctHeader, bytes)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
dgst, err := digest.ParseDigest(digestHeader)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Digest = dgst
lengthHeader := headers.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Length header")
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Size = length
return desc, nil
}
// Get issues a HEAD request for a Manifest against its named endpoint in order
// to construct a descriptor for the tag. If the registry doesn't support HEADing
// a manifest, fallback to GET.
func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
ref, err := reference.WithTag(t.name, tag)
if err != nil {
return distribution.Descriptor{}, err
}
u, err := t.ub.BuildManifestURL(ref)
if err != nil {
return distribution.Descriptor{}, err
}
req, err := http.NewRequest("HEAD", u, nil)
if err != nil {
return distribution.Descriptor{}, err
}
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
var attempts int
resp, err := t.client.Do(req)
check:
if err != nil {
return distribution.Descriptor{}, err
}
defer resp.Body.Close()
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
return descriptorFromResponse(resp)
case resp.StatusCode == http.StatusMethodNotAllowed:
req, err = http.NewRequest("GET", u, nil)
if err != nil {
return distribution.Descriptor{}, err
}
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
resp, err = t.client.Do(req)
attempts++
if attempts > 1 {
return distribution.Descriptor{}, err
}
goto check
default:
return distribution.Descriptor{}, HandleErrorResponse(resp)
}
}
func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
panic("not implemented")
}
func (t *tags) Untag(ctx context.Context, tag string) error {
panic("not implemented")
}
type manifests struct {
name reference.Named
ub *v2.URLBuilder
client *http.Client
etags map[string]string
}
func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
ref, err := reference.WithDigest(ms.name, dgst)
if err != nil {
return false, err
}
u, err := ms.ub.BuildManifestURL(ref)
if err != nil {
return false, err
}
resp, err := ms.client.Head(u)
if err != nil {
return false, err
}
if SuccessStatus(resp.StatusCode) {
return true, nil
} else if resp.StatusCode == http.StatusNotFound {
return false, nil
}
return false, HandleErrorResponse(resp)
}
// AddEtagToTag allows a client to supply an eTag to Get which will be
// used for a conditional HTTP request. If the eTag matches, a nil manifest
// and ErrManifestNotModified error will be returned. etag is automatically
// quoted when added to this map.
func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption {
return etagOption{tag, etag}
}
type etagOption struct{ tag, etag string }
func (o etagOption) Apply(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
// ReturnContentDigest allows a client to set a the content digest on
// a successful request from the 'Docker-Content-Digest' header. This
// returned digest is represents the digest which the registry uses
// to refer to the content and can be used to delete the content.
func ReturnContentDigest(dgst *digest.Digest) distribution.ManifestServiceOption {
return contentDigestOption{dgst}
}
type contentDigestOption struct{ digest *digest.Digest }
func (o contentDigestOption) Apply(ms distribution.ManifestService) error {
return nil
}
func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
var (
digestOrTag string
ref reference.Named
err error
contentDgst *digest.Digest
)
for _, option := range options {
if opt, ok := option.(distribution.WithTagOption); ok {
digestOrTag = opt.Tag
ref, err = reference.WithTag(ms.name, opt.Tag)
if err != nil {
return nil, err
}
} else if opt, ok := option.(contentDigestOption); ok {
contentDgst = opt.digest
} else {
err := option.Apply(ms)
if err != nil {
return nil, err
}
}
}
if digestOrTag == "" {
digestOrTag = dgst.String()
ref, err = reference.WithDigest(ms.name, dgst)
if err != nil {
return nil, err
}
}
u, err := ms.ub.BuildManifestURL(ref)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
if _, ok := ms.etags[digestOrTag]; ok {
req.Header.Set("If-None-Match", ms.etags[digestOrTag])
}
resp, err := ms.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified {
return nil, distribution.ErrManifestNotModified
} else if SuccessStatus(resp.StatusCode) {
if contentDgst != nil {
dgst, err := digest.ParseDigest(resp.Header.Get("Docker-Content-Digest"))
if err == nil {
*contentDgst = dgst
}
}
mt := resp.Header.Get("Content-Type")
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
m, _, err := distribution.UnmarshalManifest(mt, body)
if err != nil {
return nil, err
}
return m, nil
}
return nil, HandleErrorResponse(resp)
}
// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the
// tag name in order to build the correct upload URL.
func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
ref := ms.name
var tagged bool
for _, option := range options {
if opt, ok := option.(distribution.WithTagOption); ok {
var err error
ref, err = reference.WithTag(ref, opt.Tag)
if err != nil {
return "", err
}
tagged = true
} else {
err := option.Apply(ms)
if err != nil {
return "", err
}
}
}
mediaType, p, err := m.Payload()
if err != nil {
return "", err
}
if !tagged {
// generate a canonical digest and Put by digest
_, d, err := distribution.UnmarshalManifest(mediaType, p)
if err != nil {
return "", err
}
ref, err = reference.WithDigest(ref, d.Digest)
if err != nil {
return "", err
}
}
manifestURL, err := ms.ub.BuildManifestURL(ref)
if err != nil {
return "", err
}
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p))
if err != nil {
return "", err
}
putRequest.Header.Set("Content-Type", mediaType)
resp, err := ms.client.Do(putRequest)
if err != nil {
return "", err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
dgstHeader := resp.Header.Get("Docker-Content-Digest")
dgst, err := digest.ParseDigest(dgstHeader)
if err != nil {
return "", err
}
return dgst, nil
}
return "", HandleErrorResponse(resp)
}
func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error {
ref, err := reference.WithDigest(ms.name, dgst)
if err != nil {
return err
}
u, err := ms.ub.BuildManifestURL(ref)
if err != nil {
return err
}
req, err := http.NewRequest("DELETE", u, nil)
if err != nil {
return err
}
resp, err := ms.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
return nil
}
return HandleErrorResponse(resp)
}
// todo(richardscothern): Restore interface and implementation with merge of #1050
/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
panic("not supported")
}*/
type blobs struct {
name reference.Named
ub *v2.URLBuilder
client *http.Client
statter distribution.BlobDescriptorService
distribution.BlobDeleter
}
func sanitizeLocation(location, base string) (string, error) {
baseURL, err := url.Parse(base)
if err != nil {
return "", err
}
locationURL, err := url.Parse(location)
if err != nil {
return "", err
}
return baseURL.ResolveReference(locationURL).String(), nil
}
func (bs *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
return bs.statter.Stat(ctx, dgst)
}
func (bs *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
reader, err := bs.Open(ctx, dgst)
if err != nil {
return nil, err
}
defer reader.Close()
return ioutil.ReadAll(reader)
}
func (bs *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
ref, err := reference.WithDigest(bs.name, dgst)
if err != nil {
return nil, err
}
blobURL, err := bs.ub.BuildBlobURL(ref)
if err != nil {
return nil, err
}
return transport.NewHTTPReadSeeker(bs.client, blobURL,
func(resp *http.Response) error {
if resp.StatusCode == http.StatusNotFound {
return distribution.ErrBlobUnknown
}
return HandleErrorResponse(resp)
}), nil
}
func (bs *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
panic("not implemented")
}
func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
writer, err := bs.Create(ctx)
if err != nil {
return distribution.Descriptor{}, err
}
dgstr := digest.Canonical.New()
n, err := io.Copy(writer, io.TeeReader(bytes.NewReader(p), dgstr.Hash()))
if err != nil {
return distribution.Descriptor{}, err
}
if n < int64(len(p)) {
return distribution.Descriptor{}, fmt.Errorf("short copy: wrote %d of %d", n, len(p))
}
desc := distribution.Descriptor{
MediaType: mediaType,
Size: int64(len(p)),
Digest: dgstr.Digest(),
}
return writer.Commit(ctx, desc)
}
// createOptions is a collection of blob creation modifiers relevant to general
// blob storage intended to be configured by the BlobCreateOption.Apply method.
type createOptions struct {
Mount struct {
ShouldMount bool
From reference.Canonical
}
}
type optionFunc func(interface{}) error
func (f optionFunc) Apply(v interface{}) error {
return f(v)
}
// WithMountFrom returns a BlobCreateOption which designates that the blob should be
// mounted from the given canonical reference.
func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption {
return optionFunc(func(v interface{}) error {
opts, ok := v.(*createOptions)
if !ok {
return fmt.Errorf("unexpected options type: %T", v)
}
opts.Mount.ShouldMount = true
opts.Mount.From = ref
return nil
})
}
func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
var opts createOptions
for _, option := range options {
err := option.Apply(&opts)
if err != nil {
return nil, err
}
}
var values []url.Values
if opts.Mount.ShouldMount {
values = append(values, url.Values{"from": {opts.Mount.From.Name()}, "mount": {opts.Mount.From.Digest().String()}})
}
u, err := bs.ub.BuildBlobUploadURL(bs.name, values...)
if err != nil {
return nil, err
}
resp, err := bs.client.Post(u, "", nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusCreated:
desc, err := bs.statter.Stat(ctx, opts.Mount.From.Digest())
if err != nil {
return nil, err
}
return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc}
case http.StatusAccepted:
// TODO(dmcgowan): Check for invalid UUID
uuid := resp.Header.Get("Docker-Upload-UUID")
location, err := sanitizeLocation(resp.Header.Get("Location"), u)
if err != nil {
return nil, err
}
return &httpBlobUpload{
statter: bs.statter,
client: bs.client,
uuid: uuid,
startedAt: time.Now(),
location: location,
}, nil
default:
return nil, HandleErrorResponse(resp)
}
}
func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
panic("not implemented")
}
func (bs *blobs) Delete(ctx context.Context, dgst digest.Digest) error {
return bs.statter.Clear(ctx, dgst)
}
type blobStatter struct {
name reference.Named
ub *v2.URLBuilder
client *http.Client
}
func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
ref, err := reference.WithDigest(bs.name, dgst)
if err != nil {
return distribution.Descriptor{}, err
}
u, err := bs.ub.BuildBlobURL(ref)
if err != nil {
return distribution.Descriptor{}, err
}
resp, err := bs.client.Head(u)
if err != nil {
return distribution.Descriptor{}, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
lengthHeader := resp.Header.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, fmt.Errorf("missing content-length header for request: %s", u)
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, fmt.Errorf("error parsing content-length: %v", err)
}
return distribution.Descriptor{
MediaType: resp.Header.Get("Content-Type"),
Size: length,
Digest: dgst,
}, nil
} else if resp.StatusCode == http.StatusNotFound {
return distribution.Descriptor{}, distribution.ErrBlobUnknown
}
return distribution.Descriptor{}, HandleErrorResponse(resp)
}
func buildCatalogValues(maxEntries int, last string) url.Values {
values := url.Values{}
if maxEntries > 0 {
values.Add("n", strconv.Itoa(maxEntries))
}
if last != "" {
values.Add("last", last)
}
return values
}
func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
ref, err := reference.WithDigest(bs.name, dgst)
if err != nil {
return err
}
blobURL, err := bs.ub.BuildBlobURL(ref)
if err != nil {
return err
}
req, err := http.NewRequest("DELETE", blobURL, nil)
if err != nil {
return err
}
resp, err := bs.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
return nil
}
return HandleErrorResponse(resp)
}
func (bs *blobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
return nil
}

1182
docs/client/repository_test.go
View file

File diff suppressed because it is too large Load diff

250
docs/client/transport/http_reader.go
View file

@ -1,250 +0,0 @@
package transport
import (
"errors"
"fmt"
"io"
"net/http"
"os"
"regexp"
"strconv"
)
var (
contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
// ErrWrongCodeForByteRange is returned if the client sends a request
// with a Range header but the server returns a 2xx or 3xx code other
// than 206 Partial Content.
ErrWrongCodeForByteRange = errors.New("expected HTTP 206 from byte range request")
)
// ReadSeekCloser combines io.ReadSeeker with io.Closer.
type ReadSeekCloser interface {
io.ReadSeeker
io.Closer
}
// NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET
// request. When seeking and starting a read from a non-zero offset
// the a "Range" header will be added which sets the offset.
// TODO(dmcgowan): Move this into a separate utility package
func NewHTTPReadSeeker(client *http.Client, url string, errorHandler func(*http.Response) error) ReadSeekCloser {
return &httpReadSeeker{
client: client,
url: url,
errorHandler: errorHandler,
}
}
type httpReadSeeker struct {
client *http.Client
url string
// errorHandler creates an error from an unsuccessful HTTP response.
// This allows the error to be created with the HTTP response body
// without leaking the body through a returned error.
errorHandler func(*http.Response) error
size int64
// rc is the remote read closer.
rc io.ReadCloser
// readerOffset tracks the offset as of the last read.
readerOffset int64
// seekOffset allows Seek to override the offset. Seek changes
// seekOffset instead of changing readOffset directly so that
// connection resets can be delayed and possibly avoided if the
// seek is undone (i.e. seeking to the end and then back to the
// beginning).
seekOffset int64
err error
}
func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) {
if hrs.err != nil {
return 0, hrs.err
}
// If we sought to a different position, we need to reset the
// connection. This logic is here instead of Seek so that if
// a seek is undone before the next read, the connection doesn't
// need to be closed and reopened. A common example of this is
// seeking to the end to determine the length, and then seeking
// back to the original position.
if hrs.readerOffset != hrs.seekOffset {
hrs.reset()
}
hrs.readerOffset = hrs.seekOffset
rd, err := hrs.reader()
if err != nil {
return 0, err
}
n, err = rd.Read(p)
hrs.seekOffset += int64(n)
hrs.readerOffset += int64(n)
return n, err
}
func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) {
if hrs.err != nil {
return 0, hrs.err
}
lastReaderOffset := hrs.readerOffset
if whence == os.SEEK_SET && hrs.rc == nil {
// If no request has been made yet, and we are seeking to an
// absolute position, set the read offset as well to avoid an
// unnecessary request.
hrs.readerOffset = offset
}
_, err := hrs.reader()
if err != nil {
hrs.readerOffset = lastReaderOffset
return 0, err
}
newOffset := hrs.seekOffset
switch whence {
case os.SEEK_CUR:
newOffset += offset
case os.SEEK_END:
if hrs.size < 0 {
return 0, errors.New("content length not known")
}
newOffset = hrs.size + offset
case os.SEEK_SET:
newOffset = offset
}
if newOffset < 0 {
err = errors.New("cannot seek to negative position")
} else {
hrs.seekOffset = newOffset
}
return hrs.seekOffset, err
}
func (hrs *httpReadSeeker) Close() error {
if hrs.err != nil {
return hrs.err
}
// close and release reader chain
if hrs.rc != nil {
hrs.rc.Close()
}
hrs.rc = nil
hrs.err = errors.New("httpLayer: closed")
return nil
}
func (hrs *httpReadSeeker) reset() {
if hrs.err != nil {
return
}
if hrs.rc != nil {
hrs.rc.Close()
hrs.rc = nil
}
}
func (hrs *httpReadSeeker) reader() (io.Reader, error) {
if hrs.err != nil {
return nil, hrs.err
}
if hrs.rc != nil {
return hrs.rc, nil
}
req, err := http.NewRequest("GET", hrs.url, nil)
if err != nil {
return nil, err
}
if hrs.readerOffset > 0 {
// If we are at different offset, issue a range request from there.
req.Header.Add("Range", fmt.Sprintf("bytes=%d-", hrs.readerOffset))
// TODO: get context in here
// context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range"))
}
resp, err := hrs.client.Do(req)
if err != nil {
return nil, err
}
// Normally would use client.SuccessStatus, but that would be a cyclic
// import
if resp.StatusCode >= 200 && resp.StatusCode <= 399 {
if hrs.readerOffset > 0 {
if resp.StatusCode != http.StatusPartialContent {
return nil, ErrWrongCodeForByteRange
}
contentRange := resp.Header.Get("Content-Range")
if contentRange == "" {
return nil, errors.New("no Content-Range header found in HTTP 206 response")
}
submatches := contentRangeRegexp.FindStringSubmatch(contentRange)
if len(submatches) < 4 {
return nil, fmt.Errorf("could not parse Content-Range header: %s", contentRange)
}
startByte, err := strconv.ParseUint(submatches[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse start of range in Content-Range header: %s", contentRange)
}
if startByte != uint64(hrs.readerOffset) {
return nil, fmt.Errorf("received Content-Range starting at offset %d instead of requested %d", startByte, hrs.readerOffset)
}
endByte, err := strconv.ParseUint(submatches[2], 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse end of range in Content-Range header: %s", contentRange)
}
if submatches[3] == "*" {
hrs.size = -1
} else {
size, err := strconv.ParseUint(submatches[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse total size in Content-Range header: %s", contentRange)
}
if endByte+1 != size {
return nil, fmt.Errorf("range in Content-Range stops before the end of the content: %s", contentRange)
}
hrs.size = int64(size)
}
} else if resp.StatusCode == http.StatusOK {
hrs.size = resp.ContentLength
} else {
hrs.size = -1
}
hrs.rc = resp.Body
} else {
defer resp.Body.Close()
if hrs.errorHandler != nil {
return nil, hrs.errorHandler(resp)
}
return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status)
}
return hrs.rc, nil
}

147
docs/client/transport/transport.go
View file

@ -1,147 +0,0 @@
package transport
import (
"io"
"net/http"
"sync"
)
// RequestModifier represents an object which will do an inplace
// modification of an HTTP request.
type RequestModifier interface {
ModifyRequest(*http.Request) error
}
type headerModifier http.Header
// NewHeaderRequestModifier returns a new RequestModifier which will
// add the given headers to a request.
func NewHeaderRequestModifier(header http.Header) RequestModifier {
return headerModifier(header)
}
func (h headerModifier) ModifyRequest(req *http.Request) error {
for k, s := range http.Header(h) {
req.Header[k] = append(req.Header[k], s...)
}
return nil
}
// NewTransport creates a new transport which will apply modifiers to
// the request on a RoundTrip call.
func NewTransport(base http.RoundTripper, modifiers ...RequestModifier) http.RoundTripper {
return &transport{
Modifiers: modifiers,
Base: base,
}
}
// transport is an http.RoundTripper that makes HTTP requests after
// copying and modifying the request
type transport struct {
Modifiers []RequestModifier
Base http.RoundTripper
mu sync.Mutex // guards modReq
modReq map[*http.Request]*http.Request // original -> modified
}
// RoundTrip authorizes and authenticates the request with an
// access token. If no token exists or token is expired,
// tries to refresh/fetch a new token.
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
req2 := cloneRequest(req)
for _, modifier := range t.Modifiers {
if err := modifier.ModifyRequest(req2); err != nil {
return nil, err
}
}
t.setModReq(req, req2)
res, err := t.base().RoundTrip(req2)
if err != nil {
t.setModReq(req, nil)
return nil, err
}
res.Body = &onEOFReader{
rc: res.Body,
fn: func() { t.setModReq(req, nil) },
}
return res, nil
}
// CancelRequest cancels an in-flight request by closing its connection.
func (t *transport) CancelRequest(req *http.Request) {
type canceler interface {
CancelRequest(*http.Request)
}
if cr, ok := t.base().(canceler); ok {
t.mu.Lock()
modReq := t.modReq[req]
delete(t.modReq, req)
t.mu.Unlock()
cr.CancelRequest(modReq)
}
}
func (t *transport) base() http.RoundTripper {
if t.Base != nil {
return t.Base
}
return http.DefaultTransport
}
func (t *transport) setModReq(orig, mod *http.Request) {
t.mu.Lock()
defer t.mu.Unlock()
if t.modReq == nil {
t.modReq = make(map[*http.Request]*http.Request)
}
if mod == nil {
delete(t.modReq, orig)
} else {
t.modReq[orig] = mod
}
}
// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
// shallow copy of the struct
r2 := new(http.Request)
*r2 = *r
// deep copy of the Header
r2.Header = make(http.Header, len(r.Header))
for k, s := range r.Header {
r2.Header[k] = append([]string(nil), s...)
}
return r2
}
type onEOFReader struct {
rc io.ReadCloser
fn func()
}
func (r *onEOFReader) Read(p []byte) (n int, err error) {
n, err = r.rc.Read(p)
if err == io.EOF {
r.runFunc()
}
return
}
func (r *onEOFReader) Close() error {
err := r.rc.Close()
r.runFunc()
return err
}
func (r *onEOFReader) runFunc() {
if fn := r.fn; fn != nil {
fn()
r.fn = nil
}
}

84
docs/compatibility.md Normal file
View file

@ -0,0 +1,84 @@
<!--[metadata]>
+++
title = "Compatibility"
description = "describes get by digest pitfall"
keywords = ["registry, manifest, images, tags, repository, distribution, digest"]
[menu.main]
parent="smn_registry_ref"
weight=9
+++
<![end-metadata]-->
# Registry Compatibility
## Synopsis
*If a manifest is pulled by _digest_ from a registry 2.3 with Docker Engine 1.9
and older, and the manifest was pushed with Docker Engine 1.10, a security check
will cause the Engine to receive a manifest it cannot use and the pull will fail.*
## Registry Manifest Support
Historically, the registry has supported a [single manifest type](./spec/manifest-v2-1.md)
known as _Schema 1_.
With the move toward multiple architecture images the distribution project
introduced two new manifest types: Schema 2 manifests and manifest lists. The
registry 2.3 supports all three manifest types and in order to be compatible
with older Docker engines will, in certain cases, do an on-the-fly
transformation of a manifest before serving the JSON in the response.
This conversion has some implications for pulling manifests by digest and this
document enumerate these implications.
## Content Addressable Storage (CAS)
Manifests are stored and retrieved in the registry by keying off a digest
representing a hash of the contents. One of the advantages provided by CAS is
security: if the contents are changed, then the digest will no longer match.
This prevents any modification of the manifest by a MITM attack or an untrusted
third party.
When a manifest is stored by the registry, this digest is returned in the HTTP
response headers and, if events are configured, delivered within the event. The
manifest can either be retrieved by the tag, or this digest.
For registry versions 2.2.1 and below, the registry will always store and
serve _Schema 1_ manifests. The Docker Engine 1.10 will first
attempt to send a _Schema 2_ manifest, falling back to sending a
Schema 1 type manifest when it detects that the registry does not
support the new version.
## Registry v2.3
### Manifest Push with Docker 1.9 and Older
The Docker Engine will construct a _Schema 1_ manifest which the
registry will persist to disk.
When the manifest is pulled by digest or tag with any docker version, a
_Schema 1_ manifest will be returned.
### Manifest Push with Docker 1.10
The docker engine will construct a _Schema 2_ manifest which the
registry will persist to disk.
When the manifest is pulled by digest or tag with Docker Engine 1.10, a
_Schema 2_ manifest will be returned. The Docker Engine 1.10
understands the new manifest format.
When the manifest is pulled by *tag* with Docker Engine 1.9 and older, the
manifest is converted on-the-fly to _Schema 1_ and sent in the
response. The Docker Engine 1.9 is compatible with this older format.
*When the manifest is pulled by _digest_ with Docker Engine 1.9 and older, the
same rewriting process will not happen in the registry. If this were to happen
the digest would no longer match the hash of the manifest and would violate the
constraints of CAS.*
For this reason if a manifest is pulled by _digest_ from a registry 2.3 with Docker
Engine 1.9 and older, and the manifest was pushed with Docker Engine 1.10, a
security check will cause the Engine to receive a manifest it cannot use and the
pull will fail.

1877
docs/configuration.md Normal file
View file

File diff suppressed because it is too large Load diff

237
docs/deploying.md Normal file
View file

@ -0,0 +1,237 @@
<!--[metadata]>
+++
title = "Deploying a registry server"
description = "Explains how to deploy a registry"
keywords = ["registry, on-prem, images, tags, repository, distribution, deployment"]
[menu.main]
parent="smn_registry"
weight=3
+++
<![end-metadata]-->
# Deploying a registry server
You need to [install Docker version 1.6.0 or newer](/engine/installation/index.md).
## Running on localhost
Start your registry:
docker run -d -p 5000:5000 --restart=always --name registry registry:2
You can now use it with docker.
Get any image from the hub and tag it to point to your registry:
docker pull ubuntu && docker tag ubuntu localhost:5000/ubuntu
... then push it to your registry:
docker push localhost:5000/ubuntu
... then pull it back from your registry:
docker pull localhost:5000/ubuntu
To stop your registry, you would:
docker stop registry && docker rm -v registry
## Storage
By default, your registry data is persisted as a [docker volume](/engine/tutorials/dockervolumes.md) on the host filesystem. Properly understanding volumes is essential if you want to stick with a local filesystem storage.
Specifically, you might want to point your volume location to a specific place in order to more easily access your registry data. To do so you can:
docker run -d -p 5000:5000 --restart=always --name registry \
-v `pwd`/data:/var/lib/registry \
registry:2
### Alternatives
You should usually consider using [another storage backend](./storage-drivers/index.md) instead of the local filesystem. Use the [storage configuration options](./configuration.md#storage) to configure an alternate storage backend.
Using one of these will allow you to more easily scale your registry, and leverage your storage redundancy and availability features.
## Running a domain registry
While running on `localhost` has its uses, most people want their registry to be more widely available. To do so, the Docker engine requires you to secure it using TLS, which is conceptually very similar to configuring your web server with SSL.
### Get a certificate
Assuming that you own the domain `myregistrydomain.com`, and that its DNS record points to the host where you are running your registry, you first need to get a certificate from a CA.
Create a `certs` directory:
mkdir -p certs
Then move and/or rename your crt file to: `certs/domain.crt`, and your key file to: `certs/domain.key`.
Make sure you stopped your registry from the previous steps, then start your registry again with TLS enabled:
docker run -d -p 5000:5000 --restart=always --name registry \
-v `pwd`/certs:/certs \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
-e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
registry:2
You should now be able to access your registry from another docker host:
docker pull ubuntu
docker tag ubuntu myregistrydomain.com:5000/ubuntu
docker push myregistrydomain.com:5000/ubuntu
docker pull myregistrydomain.com:5000/ubuntu
#### Gotcha
A certificate issuer may supply you with an *intermediate* certificate. In this case, you must combine your certificate with the intermediate's to form a *certificate bundle*. You can do this using the `cat` command:
cat domain.crt intermediate-certificates.pem > certs/domain.crt
### Let's Encrypt
The registry supports using Let's Encrypt to automatically obtain a browser-trusted certificate. For more
information on Let's Encrypt, see [https://letsencrypt.org/how-it-works/](https://letsencrypt.org/how-it-works/) and the relevant section of the [registry configuration](configuration.md#letsencrypt).
### Alternatives
While rarely advisable, you may want to use self-signed certificates instead, or use your registry in an insecure fashion. You will find instructions [here](insecure.md).
## Load Balancing Considerations
One may want to use a load balancer to distribute load, terminate TLS or
provide high availability. While a full load balancing setup is outside the
scope of this document, there are a few considerations that can make the process
smoother.
The most important aspect is that a load balanced cluster of registries must
share the same resources. For the current version of the registry, this means
the following must be the same:
- Storage Driver
- HTTP Secret
- Redis Cache (if configured)
If any of these are different, the registry will have trouble serving requests.
As an example, if you're using the filesystem driver, all registry instances
must have access to the same filesystem root, which means they should be in
the same machine. For other drivers, such as s3 or azure, they should be
accessing the same resource, and will likely share an identical configuration.
The _HTTP Secret_ coordinates uploads, so also must be the same across
instances. Configuring different redis instances will work (at the time
of writing), but will not be optimal if the instances are not shared, causing
more requests to be directed to the backend.
#### Important/Required HTTP-Headers
Getting the headers correct is very important. For all responses to any
request under the "/v2/" url space, the `Docker-Distribution-API-Version`
header should be set to the value "registry/2.0", even for a 4xx response.
This header allows the docker engine to quickly resolve authentication realms
and fallback to version 1 registries, if necessary. Confirming this is setup
correctly can help avoid problems with fallback.
In the same train of thought, you must make sure you are properly sending the
`X-Forwarded-Proto`, `X-Forwarded-For` and `Host` headers to their "client-side"
values. Failure to do so usually makes the registry issue redirects to internal
hostnames or downgrading from https to http.
A properly secured registry should return 401 when the "/v2/" endpoint is hit
without credentials. The response should include a `WWW-Authenticate`
challenge, providing guidance on how to authenticate, such as with basic auth
or a token service. If the load balancer has health checks, it is recommended
to configure it to consider a 401 response as healthy and any other as down.
This will secure your registry by ensuring that configuration problems with
authentication don't accidentally expose an unprotected registry. If you're
using a less sophisticated load balancer, such as Amazon's Elastic Load
Balancer, that doesn't allow one to change the healthy response code, health
checks can be directed at "/", which will always return a `200 OK` response.
## Restricting access
Except for registries running on secure local networks, registries should always implement access restrictions.
### Native basic auth
The simplest way to achieve access restriction is through basic authentication (this is very similar to other web servers' basic authentication mechanism).
> **Warning**: You **cannot** use authentication with an insecure registry. You have to [configure TLS first](#running-a-domain-registry) for this to work.
First create a password file with one entry for the user "testuser", with password "testpassword":
mkdir auth
docker run --entrypoint htpasswd registry:2 -Bbn testuser testpassword > auth/htpasswd
Make sure you stopped your registry from the previous step, then start it again:
docker run -d -p 5000:5000 --restart=always --name registry \
-v `pwd`/auth:/auth \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-v `pwd`/certs:/certs \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
-e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
registry:2
You should now be able to:
docker login myregistrydomain.com:5000
And then push and pull images as an authenticated user.
#### Gotcha
Seeing X509 errors is usually a sign you are trying to use self-signed certificates, and failed to [configure your docker daemon properly](insecure.md).
### Alternatives
1. You may want to leverage more advanced basic auth implementations through a proxy design, in front of the registry. You will find examples of such patterns in the [recipes list](recipes/index.md).
2. Alternatively, the Registry also supports delegated authentication, redirecting users to a specific, trusted token server. That approach requires significantly more investment, and only makes sense if you want to fully configure ACLs and more control over the Registry integration into your global authorization and authentication systems.
You will find [background information here](spec/auth/token.md), and [configuration information here](configuration.md#auth).
Beware that you will have to implement your own authentication service for this to work, or leverage a third-party implementation.
## Managing with Compose
As your registry configuration grows more complex, dealing with it can quickly become tedious.
It's highly recommended to use [Docker Compose](/compose/index.md) to facilitate operating your registry.
Here is a simple `docker-compose.yml` example that condenses everything explained so far:
```
registry:
restart: always
image: registry:2
ports:
- 5000:5000
environment:
REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
REGISTRY_HTTP_TLS_KEY: /certs/domain.key
REGISTRY_AUTH: htpasswd
REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
volumes:
- /path/data:/var/lib/registry
- /path/certs:/certs
- /path/auth:/auth
```
> **Warning**: replace `/path` by whatever directory that holds your `certs` and `auth` folder from above.
You can then start your registry with a simple
docker-compose up -d
## Next
You will find more specific and advanced informations in the following sections:
- [Configuration reference](configuration.md)
- [Working with notifications](notifications.md)
- [Advanced "recipes"](recipes/index.md)
- [Registry API](spec/api.md)
- [Storage driver model](storage-drivers/index.md)
- [Token authentication](spec/auth/token.md)

27
docs/deprecated.md Normal file
View file

@ -0,0 +1,27 @@
<!--[metadata]>
+++
title = "Deprecated Features"
description = "describes deprecated functionality"
keywords = ["registry, manifest, images, signatures, repository, distribution, digest"]
[menu.main]
parent="smn_registry_ref"
weight=8
+++
<![end-metadata]-->
# Docker Registry Deprecation
This document details functionality or components which are deprecated within
the registry.
### v2.5.0
The signature store has been removed from the registry. Since `v2.4.0` it has
been possible to configure the registry to generate manifest signatures rather
than load them from storage. In this version of the registry this becomes
the default behavior. Signatures which are attached to manifests on put are
not stored in the registry. This does not alter the functional behavior of
the registry.
Old signatures blobs can be removed from the registry storage by running the
garbage-collect subcommand.

2
docs/doc.go
View file

@ -1,2 +0,0 @@
// Package registry provides the main entrypoints for running a registry.
package registry

137
docs/garbage-collection.md Normal file
View file

@ -0,0 +1,137 @@
<!--[metadata]>
+++
title = "Garbage Collection"
description = "High level discussion of garbage collection"
keywords = ["registry, garbage, images, tags, repository, distribution"]
[menu.main]
parent="smn_registry_ref"
weight=4
+++
<![end-metadata]-->
# Garbage Collection
As of v2.4.0 a garbage collector command is included within the registry binary.
This document describes what this command does and how and why it should be used.
## What is Garbage Collection?
From [wikipedia](https://en.wikipedia.org/wiki/Garbage_collection_(computer_science)):
"In computer science, garbage collection (GC) is a form of automatic memory management. The
garbage collector, or just collector, attempts to reclaim garbage, or memory occupied by
objects that are no longer in use by the program."
In the context of the Docker registry, garbage collection is the process of
removing blobs from the filesystem which are no longer referenced by a
manifest. Blobs can include both layers and manifests.
## Why Garbage Collection?
Registry data can occupy considerable amounts of disk space and freeing up
this disk space is an oft-requested feature. Additionally for reasons of security it
can be desirable to ensure that certain layers no longer exist on the filesystem.
## Garbage Collection in the Registry
Filesystem layers are stored by their content address in the Registry. This
has many advantages, one of which is that data is stored once and referred to by manifests.
See [here](compatibility.md#content-addressable-storage-cas) for more details.
Layers are therefore shared amongst manifests; each manifest maintains a reference
to the layer. As long as a layer is referenced by one manifest, it cannot be garbage
collected.
Manifests and layers can be 'deleted` with the registry API (refer to the API
documentation [here](spec/api.md#deleting-a-layer) and
[here](spec/api.md#deleting-an-image) for details). This API removes references
to the target and makes them eligible for garbage collection. It also makes them
unable to be read via the API.
If a layer is deleted it will be removed from the filesystem when garbage collection
is run. If a manifest is deleted the layers to which it refers will be removed from
the filesystem if no other manifests refers to them.
### Example
In this example manifest A references two layers: `a` and `b`. Manifest `B` references
layers `a` and `c`. In this state, nothing is eligible for garbage collection:
```
A -----> a <----- B
\--> b |
c <--/
```
Manifest B is deleted via the API:
```
A -----> a B
\--> b
c
```
In this state layer `c` no longer has a reference and is eligible for garbage
collection. Layer `a` had one reference removed but will not be garbage
collected as it is still referenced by manifest `A`. The blob representing
manifest `B` will also be eligible for garbage collection.
After garbage collection has been run manifest `A` and its blobs remain.
```
A -----> a
\--> b
```
## How Garbage Collection works
Garbage collection runs in two phases. First, in the 'mark' phase, the process
scans all the manifests in the registry. From these manifests, it constructs a
set of content address digests. This set is the 'mark set' and denotes the set
of blobs to *not* delete. Secondly, in the 'sweep' phase, the process scans all
the blobs and if a blob's content address digest is not in the mark set, the
process will delete it.
> **NOTE** You should ensure that the registry is in read-only mode or not running at
> all. If you were to upload an image while garbage collection is running, there is the
> risk that the image's layers will be mistakenly deleted, leading to a corrupted image.
This type of garbage collection is known as stop-the-world garbage collection. In future
registry versions the intention is that garbage collection will be an automated background
action and this manual process will no longer apply.
# Running garbage collection
Garbage collection can be run as follows
`bin/registry garbage-collect [--dry-run] /path/to/config.yml`
The garbage-collect command accepts a `--dry-run` parameter, which will print the progress
of the mark and sweep phases without removing any data. Running with a log leve of `info`
will give a clear indication of what will and will not be deleted.
_Sample output from a dry run garbage collection with registry log level set to `info`_
```
hello-world
hello-world: marking manifest sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf
hello-world: marking blob sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb
hello-world: marking blob sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4
hello-world: marking configuration sha256:690ed74de00f99a7d00a98a5ad855ac4febd66412be132438f9b8dbd300a937d
ubuntu
4 blobs marked, 5 blobs eligible for deletion
blob eligible for deletion: sha256:28e09fddaacbfc8a13f82871d9d66141a6ed9ca526cb9ed295ef545ab4559b81
blob eligible for deletion: sha256:7e15ce58ccb2181a8fced7709e9893206f0937cc9543bc0c8178ea1cf4d7e7b5
blob eligible for deletion: sha256:87192bdbe00f8f2a62527f36bb4c7c7f4eaf9307e4b87e8334fb6abec1765bcb
blob eligible for deletion: sha256:b549a9959a664038fc35c155a95742cf12297672ca0ae35735ec027d55bf4e97
blob eligible for deletion: sha256:f251d679a7c61455f06d793e43c06786d7766c88b8c24edf242b2c08e3c3f599
```

70
docs/glossary.md Normal file
View file

@ -0,0 +1,70 @@
<!--[metadata]>
+++
draft = true
+++
<![end-metadata]-->
# Glossary
This page contains definitions for distribution related terms.
<dl>
<dt id="blob"><h4>Blob</h4></dt>
<dd>
<blockquote>A blob is any kind of content that is stored by a Registry under a content-addressable identifier (a "digest").</blockquote>
<p>
<a href="#layer">Layers</a> are a good example of "blobs".
</p>
</dd>
<dt id="image"><h4>Image</h4></dt>
<dd>
<blockquote>An image is a named set of immutable data from which a Docker container can be created.</blockquote>
<p>
An image is represented by a json file called a <a href="#manifest">manifest</a>, and is conceptually a set of <a hred="#layer">layers</a>.
Image names indicate the location where they can be pulled from and pushed to, as they usually start with a <a href="#registry">registry</a> domain name and port.
</p>
</dd>
<dt id="layer"><h4>Layer</h4></dt>
<dd>
<blockquote>A layer is a tar archive bundling partial content from a filesystem.</blockquote>
<p>
Layers from an <a href="#image">image</a> are usually extracted in order on top of each other to make up a root filesystem from which containers run out.
</p>
</dd>
<dt id="manifest"><h4>Manifest</h4></dt>
<dd><blockquote>A manifest is the JSON representation of an image.</blockquote></dd>
<dt id="namespace"><h4>Namespace</h4></dt>
<dd><blockquote>A namespace is a collection of repositories with a common name prefix.</blockquote>
<p>
The namespace with an empty prefix is considered the Global Namespace.
</p>
</dd>
<dt id="registry"><h4>Registry</h4></dt>
<dd><blockquote>A registry is a service that let you store and deliver <a href="#images">images</a>.</blockquote>
</dd>
<dt id="registry"><h4>Repository</h4></dt>
<dd>
<blockquote>A repository is a set of data containing all versions of a given image.</blockquote>
</dd>
<dt id="scope"><h4>Scope</h4></dt>
<dd><blockquote>A scope is the portion of a namespace onto which a given authorization token is granted.</blockquote></dd>
<dt id="tag"><h4>Tag</h4></dt>
<dd><blockquote>A tag is conceptually a "version" of a <a href="#image">named image</a>.</blockquote>
<p>
Example: `docker pull myimage:latest` instructs docker to pull the image "myimage" in version "latest".
</p>
</dd>
</dl>

2474
docs/handlers/api_test.go
View file

File diff suppressed because it is too large Load diff

996
docs/handlers/app.go
View file

@ -1,996 +0,0 @@
package handlers
import (
cryptorand "crypto/rand"
"expvar"
"fmt"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"runtime"
"time"
log "github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/configuration"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/health"
"github.com/docker/distribution/health/checks"
"github.com/docker/distribution/notifications"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/auth"
registrymiddleware "github.com/docker/distribution/registry/middleware/registry"
repositorymiddleware "github.com/docker/distribution/registry/middleware/repository"
"github.com/docker/distribution/registry/proxy"
"github.com/docker/distribution/registry/storage"
memorycache "github.com/docker/distribution/registry/storage/cache/memory"
rediscache "github.com/docker/distribution/registry/storage/cache/redis"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/factory"
storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
"github.com/docker/distribution/version"
"github.com/docker/libtrust"
"github.com/garyburd/redigo/redis"
"github.com/gorilla/mux"
"golang.org/x/net/context"
)
// randomSecretSize is the number of random bytes to generate if no secret
// was specified.
const randomSecretSize = 32
// defaultCheckInterval is the default time in between health checks
const defaultCheckInterval = 10 * time.Second
// App is a global registry application object. Shared resources can be placed
// on this object that will be accessible from all requests. Any writable
// fields should be protected.
type App struct {
context.Context
Config *configuration.Configuration
router *mux.Router // main application router, configured with dispatchers
driver storagedriver.StorageDriver // driver maintains the app global storage driver instance.
registry distribution.Namespace // registry is the primary registry backend for the app instance.
accessController auth.AccessController // main access controller for application
// httpHost is a parsed representation of the http.host parameter from
// the configuration. Only the Scheme and Host fields are used.
httpHost url.URL
// events contains notification related configuration.
events struct {
sink notifications.Sink
source notifications.SourceRecord
}
redis *redis.Pool
// trustKey is a deprecated key used to sign manifests converted to
// schema1 for backward compatibility. It should not be used for any
// other purposes.
trustKey libtrust.PrivateKey
// isCache is true if this registry is configured as a pull through cache
isCache bool
// readOnly is true if the registry is in a read-only maintenance mode
readOnly bool
}
// NewApp takes a configuration and returns a configured app, ready to serve
// requests. The app only implements ServeHTTP and can be wrapped in other
// handlers accordingly.
func NewApp(ctx context.Context, config *configuration.Configuration) *App {
app := &App{
Config: config,
Context: ctx,
router: v2.RouterWithPrefix(config.HTTP.Prefix),
isCache: config.Proxy.RemoteURL != "",
}
// Register the handler dispatchers.
app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler {
return http.HandlerFunc(apiBase)
})
app.register(v2.RouteNameManifest, imageManifestDispatcher)
app.register(v2.RouteNameCatalog, catalogDispatcher)
app.register(v2.RouteNameTags, tagsDispatcher)
app.register(v2.RouteNameBlob, blobDispatcher)
app.register(v2.RouteNameBlobUpload, blobUploadDispatcher)
app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher)
// override the storage driver's UA string for registry outbound HTTP requests
storageParams := config.Storage.Parameters()
if storageParams == nil {
storageParams = make(configuration.Parameters)
}
storageParams["useragent"] = fmt.Sprintf("docker-distribution/%s %s", version.Version, runtime.Version())
var err error
app.driver, err = factory.Create(config.Storage.Type(), storageParams)
if err != nil {
// TODO(stevvooe): Move the creation of a service into a protected
// method, where this is created lazily. Its status can be queried via
// a health check.
panic(err)
}
purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := config.Storage["maintenance"]; ok {
if v, ok := mc["uploadpurging"]; ok {
purgeConfig, ok = v.(map[interface{}]interface{})
if !ok {
panic("uploadpurging config key must contain additional keys")
}
}
if v, ok := mc["readonly"]; ok {
readOnly, ok := v.(map[interface{}]interface{})
if !ok {
panic("readonly config key must contain additional keys")
}
if readOnlyEnabled, ok := readOnly["enabled"]; ok {
app.readOnly, ok = readOnlyEnabled.(bool)
if !ok {
panic("readonly's enabled config key must have a boolean value")
}
}
}
}
startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig)
app.driver, err = applyStorageMiddleware(app.driver, config.Middleware["storage"])
if err != nil {
panic(err)
}
app.configureSecret(config)
app.configureEvents(config)
app.configureRedis(config)
app.configureLogHook(config)
options := registrymiddleware.GetRegistryOptions()
if config.Compatibility.Schema1.TrustKey != "" {
app.trustKey, err = libtrust.LoadKeyFile(config.Compatibility.Schema1.TrustKey)
if err != nil {
panic(fmt.Sprintf(`could not load schema1 "signingkey" parameter: %v`, err))
}
} else {
// Generate an ephemeral key to be used for signing converted manifests
// for clients that don't support schema2.
app.trustKey, err = libtrust.GenerateECP256PrivateKey()
if err != nil {
panic(err)
}
}
options = append(options, storage.Schema1SigningKey(app.trustKey))
if config.HTTP.Host != "" {
u, err := url.Parse(config.HTTP.Host)
if err != nil {
panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err))
}
app.httpHost = *u
}
if app.isCache {
options = append(options, storage.DisableDigestResumption)
}
// configure deletion
if d, ok := config.Storage["delete"]; ok {
e, ok := d["enabled"]
if ok {
if deleteEnabled, ok := e.(bool); ok && deleteEnabled {
options = append(options, storage.EnableDelete)
}
}
}
// configure redirects
var redirectDisabled bool
if redirectConfig, ok := config.Storage["redirect"]; ok {
v := redirectConfig["disable"]
switch v := v.(type) {
case bool:
redirectDisabled = v
default:
panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig))
}
}
if redirectDisabled {
ctxu.GetLogger(app).Infof("backend redirection disabled")
} else {
options = append(options, storage.EnableRedirect)
}
// configure storage caches
if cc, ok := config.Storage["cache"]; ok {
v, ok := cc["blobdescriptor"]
if !ok {
// Backwards compatible: "layerinfo" == "blobdescriptor"
v = cc["layerinfo"]
}
switch v {
case "redis":
if app.redis == nil {
panic("redis configuration required to use for layerinfo cache")
}
cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis)
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using redis blob descriptor cache")
case "inmemory":
cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider()
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache")
default:
if v != "" {
ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", config.Storage["cache"])
}
}
}
if app.registry == nil {
// configure the registry if no cache section is available.
app.registry, err = storage.NewRegistry(app.Context, app.driver, options...)
if err != nil {
panic("could not create registry: " + err.Error())
}
}
app.registry, err = applyRegistryMiddleware(app, app.registry, config.Middleware["registry"])
if err != nil {
panic(err)
}
authType := config.Auth.Type()
if authType != "" {
accessController, err := auth.GetAccessController(config.Auth.Type(), config.Auth.Parameters())
if err != nil {
panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err))
}
app.accessController = accessController
ctxu.GetLogger(app).Debugf("configured %q access controller", authType)
}
// configure as a pull through cache
if config.Proxy.RemoteURL != "" {
app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, config.Proxy)
if err != nil {
panic(err.Error())
}
app.isCache = true
ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", config.Proxy.RemoteURL)
}
return app
}
// RegisterHealthChecks is an awful hack to defer health check registration
// control to callers. This should only ever be called once per registry
// process, typically in a main function. The correct way would be register
// health checks outside of app, since multiple apps may exist in the same
// process. Because the configuration and app are tightly coupled,
// implementing this properly will require a refactor. This method may panic
// if called twice in the same process.
func (app *App) RegisterHealthChecks(healthRegistries ...*health.Registry) {
if len(healthRegistries) > 1 {
panic("RegisterHealthChecks called with more than one registry")
}
healthRegistry := health.DefaultRegistry
if len(healthRegistries) == 1 {
healthRegistry = healthRegistries[0]
}
if app.Config.Health.StorageDriver.Enabled {
interval := app.Config.Health.StorageDriver.Interval
if interval == 0 {
interval = defaultCheckInterval
}
storageDriverCheck := func() error {
_, err := app.driver.List(app, "/") // "/" should always exist
return err // any error will be treated as failure
}
if app.Config.Health.StorageDriver.Threshold != 0 {
healthRegistry.RegisterPeriodicThresholdFunc("storagedriver_"+app.Config.Storage.Type(), interval, app.Config.Health.StorageDriver.Threshold, storageDriverCheck)
} else {
healthRegistry.RegisterPeriodicFunc("storagedriver_"+app.Config.Storage.Type(), interval, storageDriverCheck)
}
}
for _, fileChecker := range app.Config.Health.FileCheckers {
interval := fileChecker.Interval
if interval == 0 {
interval = defaultCheckInterval
}
ctxu.GetLogger(app).Infof("configuring file health check path=%s, interval=%d", fileChecker.File, interval/time.Second)
healthRegistry.Register(fileChecker.File, health.PeriodicChecker(checks.FileChecker(fileChecker.File), interval))
}
for _, httpChecker := range app.Config.Health.HTTPCheckers {
interval := httpChecker.Interval
if interval == 0 {
interval = defaultCheckInterval
}
statusCode := httpChecker.StatusCode
if statusCode == 0 {
statusCode = 200
}
checker := checks.HTTPChecker(httpChecker.URI, statusCode, httpChecker.Timeout, httpChecker.Headers)
if httpChecker.Threshold != 0 {
ctxu.GetLogger(app).Infof("configuring HTTP health check uri=%s, interval=%d, threshold=%d", httpChecker.URI, interval/time.Second, httpChecker.Threshold)
healthRegistry.Register(httpChecker.URI, health.PeriodicThresholdChecker(checker, interval, httpChecker.Threshold))
} else {
ctxu.GetLogger(app).Infof("configuring HTTP health check uri=%s, interval=%d", httpChecker.URI, interval/time.Second)
healthRegistry.Register(httpChecker.URI, health.PeriodicChecker(checker, interval))
}
}
for _, tcpChecker := range app.Config.Health.TCPCheckers {
interval := tcpChecker.Interval
if interval == 0 {
interval = defaultCheckInterval
}
checker := checks.TCPChecker(tcpChecker.Addr, tcpChecker.Timeout)
if tcpChecker.Threshold != 0 {
ctxu.GetLogger(app).Infof("configuring TCP health check addr=%s, interval=%d, threshold=%d", tcpChecker.Addr, interval/time.Second, tcpChecker.Threshold)
healthRegistry.Register(tcpChecker.Addr, health.PeriodicThresholdChecker(checker, interval, tcpChecker.Threshold))
} else {
ctxu.GetLogger(app).Infof("configuring TCP health check addr=%s, interval=%d", tcpChecker.Addr, interval/time.Second)
healthRegistry.Register(tcpChecker.Addr, health.PeriodicChecker(checker, interval))
}
}
}
// register a handler with the application, by route name. The handler will be
// passed through the application filters and context will be constructed at
// request time.
func (app *App) register(routeName string, dispatch dispatchFunc) {
// TODO(stevvooe): This odd dispatcher/route registration is by-product of
// some limitations in the gorilla/mux router. We are using it to keep
// routing consistent between the client and server, but we may want to
// replace it with manual routing and structure-based dispatch for better
// control over the request execution.
app.router.GetRoute(routeName).Handler(app.dispatcher(dispatch))
}
// configureEvents prepares the event sink for action.
func (app *App) configureEvents(configuration *configuration.Configuration) {
// Configure all of the endpoint sinks.
var sinks []notifications.Sink
for _, endpoint := range configuration.Notifications.Endpoints {
if endpoint.Disabled {
ctxu.GetLogger(app).Infof("endpoint %s disabled, skipping", endpoint.Name)
continue
}
ctxu.GetLogger(app).Infof("configuring endpoint %v (%v), timeout=%s, headers=%v", endpoint.Name, endpoint.URL, endpoint.Timeout, endpoint.Headers)
endpoint := notifications.NewEndpoint(endpoint.Name, endpoint.URL, notifications.EndpointConfig{
Timeout: endpoint.Timeout,
Threshold: endpoint.Threshold,
Backoff: endpoint.Backoff,
Headers: endpoint.Headers,
})
sinks = append(sinks, endpoint)
}
// NOTE(stevvooe): Moving to a new queuing implementation is as easy as
// replacing broadcaster with a rabbitmq implementation. It's recommended
// that the registry instances also act as the workers to keep deployment
// simple.
app.events.sink = notifications.NewBroadcaster(sinks...)
// Populate registry event source
hostname, err := os.Hostname()
if err != nil {
hostname = configuration.HTTP.Addr
} else {
// try to pick the port off the config
_, port, err := net.SplitHostPort(configuration.HTTP.Addr)
if err == nil {
hostname = net.JoinHostPort(hostname, port)
}
}
app.events.source = notifications.SourceRecord{
Addr: hostname,
InstanceID: ctxu.GetStringValue(app, "instance.id"),
}
}
func (app *App) configureRedis(configuration *configuration.Configuration) {
if configuration.Redis.Addr == "" {
ctxu.GetLogger(app).Infof("redis not configured")
return
}
pool := &redis.Pool{
Dial: func() (redis.Conn, error) {
// TODO(stevvooe): Yet another use case for contextual timing.
ctx := context.WithValue(app, "redis.connect.startedat", time.Now())
done := func(err error) {
logger := ctxu.GetLoggerWithField(ctx, "redis.connect.duration",
ctxu.Since(ctx, "redis.connect.startedat"))
if err != nil {
logger.Errorf("redis: error connecting: %v", err)
} else {
logger.Infof("redis: connect %v", configuration.Redis.Addr)
}
}
conn, err := redis.DialTimeout("tcp",
configuration.Redis.Addr,
configuration.Redis.DialTimeout,
configuration.Redis.ReadTimeout,
configuration.Redis.WriteTimeout)
if err != nil {
ctxu.GetLogger(app).Errorf("error connecting to redis instance %s: %v",
configuration.Redis.Addr, err)
done(err)
return nil, err
}
// authorize the connection
if configuration.Redis.Password != "" {
if _, err = conn.Do("AUTH", configuration.Redis.Password); err != nil {
defer conn.Close()
done(err)
return nil, err
}
}
// select the database to use
if configuration.Redis.DB != 0 {
if _, err = conn.Do("SELECT", configuration.Redis.DB); err != nil {
defer conn.Close()
done(err)
return nil, err
}
}
done(nil)
return conn, nil
},
MaxIdle: configuration.Redis.Pool.MaxIdle,
MaxActive: configuration.Redis.Pool.MaxActive,
IdleTimeout: configuration.Redis.Pool.IdleTimeout,
TestOnBorrow: func(c redis.Conn, t time.Time) error {
// TODO(stevvooe): We can probably do something more interesting
// here with the health package.
_, err := c.Do("PING")
return err
},
Wait: false, // if a connection is not avialable, proceed without cache.
}
app.redis = pool
// setup expvar
registry := expvar.Get("registry")
if registry == nil {
registry = expvar.NewMap("registry")
}
registry.(*expvar.Map).Set("redis", expvar.Func(func() interface{} {
return map[string]interface{}{
"Config": configuration.Redis,
"Active": app.redis.ActiveCount(),
}
}))
}
// configureLogHook prepares logging hook parameters.
func (app *App) configureLogHook(configuration *configuration.Configuration) {
entry, ok := ctxu.GetLogger(app).(*log.Entry)
if !ok {
// somehow, we are not using logrus
return
}
logger := entry.Logger
for _, configHook := range configuration.Log.Hooks {
if !configHook.Disabled {
switch configHook.Type {
case "mail":
hook := &logHook{}
hook.LevelsParam = configHook.Levels
hook.Mail = &mailer{
Addr: configHook.MailOptions.SMTP.Addr,
Username: configHook.MailOptions.SMTP.Username,
Password: configHook.MailOptions.SMTP.Password,
Insecure: configHook.MailOptions.SMTP.Insecure,
From: configHook.MailOptions.From,
To: configHook.MailOptions.To,
}
logger.Hooks.Add(hook)
default:
}
}
}
}
// configureSecret creates a random secret if a secret wasn't included in the
// configuration.
func (app *App) configureSecret(configuration *configuration.Configuration) {
if configuration.HTTP.Secret == "" {
var secretBytes [randomSecretSize]byte
if _, err := cryptorand.Read(secretBytes[:]); err != nil {
panic(fmt.Sprintf("could not generate random bytes for HTTP secret: %v", err))
}
configuration.HTTP.Secret = string(secretBytes[:])
ctxu.GetLogger(app).Warn("No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable.")
}
}
func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close() // ensure that request body is always closed.
// Instantiate an http context here so we can track the error codes
// returned by the request router.
ctx := defaultContextManager.context(app, w, r)
defer func() {
status, ok := ctx.Value("http.response.status").(int)
if ok && status >= 200 && status <= 399 {
ctxu.GetResponseLogger(ctx).Infof("response completed")
}
}()
defer defaultContextManager.release(ctx)
// NOTE(stevvooe): Total hack to get instrumented responsewriter from context.
var err error
w, err = ctxu.GetResponseWriter(ctx)
if err != nil {
ctxu.GetLogger(ctx).Warnf("response writer not found in context")
}
// Set a header with the Docker Distribution API Version for all responses.
w.Header().Add("Docker-Distribution-API-Version", "registry/2.0")
app.router.ServeHTTP(w, r)
}
// dispatchFunc takes a context and request and returns a constructed handler
// for the route. The dispatcher will use this to dynamically create request
// specific handlers for each endpoint without creating a new router for each
// request.
type dispatchFunc func(ctx *Context, r *http.Request) http.Handler
// TODO(stevvooe): dispatchers should probably have some validation error
// chain with proper error reporting.
// dispatcher returns a handler that constructs a request specific context and
// handler, using the dispatch factory function.
func (app *App) dispatcher(dispatch dispatchFunc) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for headerName, headerValues := range app.Config.HTTP.Headers {
for _, value := range headerValues {
w.Header().Add(headerName, value)
}
}
context := app.context(w, r)
if err := app.authorized(w, r, context); err != nil {
ctxu.GetLogger(context).Warnf("error authorizing context: %v", err)
return
}
// Add username to request logging
context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, auth.UserNameKey))
if app.nameRequired(r) {
nameRef, err := reference.ParseNamed(getName(context))
if err != nil {
ctxu.GetLogger(context).Errorf("error parsing reference from context: %v", err)
context.Errors = append(context.Errors, distribution.ErrRepositoryNameInvalid{
Name: getName(context),
Reason: err,
})
if err := errcode.ServeJSON(w, context.Errors); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
return
}
repository, err := app.registry.Repository(context, nameRef)
if err != nil {
ctxu.GetLogger(context).Errorf("error resolving repository: %v", err)
switch err := err.(type) {
case distribution.ErrRepositoryUnknown:
context.Errors = append(context.Errors, v2.ErrorCodeNameUnknown.WithDetail(err))
case distribution.ErrRepositoryNameInvalid:
context.Errors = append(context.Errors, v2.ErrorCodeNameInvalid.WithDetail(err))
case errcode.Error:
context.Errors = append(context.Errors, err)
}
if err := errcode.ServeJSON(w, context.Errors); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
return
}
// assign and decorate the authorized repository with an event bridge.
context.Repository = notifications.Listen(
repository,
app.eventBridge(context, r))
context.Repository, err = applyRepoMiddleware(app, context.Repository, app.Config.Middleware["repository"])
if err != nil {
ctxu.GetLogger(context).Errorf("error initializing repository middleware: %v", err)
context.Errors = append(context.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
if err := errcode.ServeJSON(w, context.Errors); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
return
}
}
dispatch(context, r).ServeHTTP(w, r)
// Automated error response handling here. Handlers may return their
// own errors if they need different behavior (such as range errors
// for layer upload).
if context.Errors.Len() > 0 {
if err := errcode.ServeJSON(w, context.Errors); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
app.logError(context, context.Errors)
}
})
}
func (app *App) logError(context context.Context, errors errcode.Errors) {
for _, e1 := range errors {
var c ctxu.Context
switch e1.(type) {
case errcode.Error:
e, _ := e1.(errcode.Error)
c = ctxu.WithValue(context, "err.code", e.Code)
c = ctxu.WithValue(c, "err.message", e.Code.Message())
c = ctxu.WithValue(c, "err.detail", e.Detail)
case errcode.ErrorCode:
e, _ := e1.(errcode.ErrorCode)
c = ctxu.WithValue(context, "err.code", e)
c = ctxu.WithValue(c, "err.message", e.Message())
default:
// just normal go 'error'
c = ctxu.WithValue(context, "err.code", errcode.ErrorCodeUnknown)
c = ctxu.WithValue(c, "err.message", e1.Error())
}
c = ctxu.WithLogger(c, ctxu.GetLogger(c,
"err.code",
"err.message",
"err.detail"))
ctxu.GetResponseLogger(c).Errorf("response completed with error")
}
}
// context constructs the context object for the application. This only be
// called once per request.
func (app *App) context(w http.ResponseWriter, r *http.Request) *Context {
ctx := defaultContextManager.context(app, w, r)
ctx = ctxu.WithVars(ctx, r)
ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx,
"vars.name",
"vars.reference",
"vars.digest",
"vars.uuid"))
context := &Context{
App: app,
Context: ctx,
}
if app.httpHost.Scheme != "" && app.httpHost.Host != "" {
// A "host" item in the configuration takes precedence over
// X-Forwarded-Proto and X-Forwarded-Host headers, and the
// hostname in the request.
context.urlBuilder = v2.NewURLBuilder(&app.httpHost, false)
} else {
context.urlBuilder = v2.NewURLBuilderFromRequest(r, app.Config.HTTP.RelativeURLs)
}
return context
}
// authorized checks if the request can proceed with access to the requested
// repository. If it succeeds, the context may access the requested
// repository. An error will be returned if access is not available.
func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Context) error {
ctxu.GetLogger(context).Debug("authorizing request")
repo := getName(context)
if app.accessController == nil {
return nil // access controller is not enabled.
}
var accessRecords []auth.Access
if repo != "" {
accessRecords = appendAccessRecords(accessRecords, r.Method, repo)
if fromRepo := r.FormValue("from"); fromRepo != "" {
// mounting a blob from one repository to another requires pull (GET)
// access to the source repository.
accessRecords = appendAccessRecords(accessRecords, "GET", fromRepo)
}
} else {
// Only allow the name not to be set on the base route.
if app.nameRequired(r) {
// For this to be properly secured, repo must always be set for a
// resource that may make a modification. The only condition under
// which name is not set and we still allow access is when the
// base route is accessed. This section prevents us from making
// that mistake elsewhere in the code, allowing any operation to
// proceed.
if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
return fmt.Errorf("forbidden: no repository name")
}
accessRecords = appendCatalogAccessRecord(accessRecords, r)
}
ctx, err := app.accessController.Authorized(context.Context, accessRecords...)
if err != nil {
switch err := err.(type) {
case auth.Challenge:
// Add the appropriate WWW-Auth header
err.SetHeaders(w)
if err := errcode.ServeJSON(w, errcode.ErrorCodeUnauthorized.WithDetail(accessRecords)); err != nil {
ctxu.GetLogger(context).Errorf("error serving error json: %v (from %v)", err, context.Errors)
}
default:
// This condition is a potential security problem either in
// the configuration or whatever is backing the access
// controller. Just return a bad request with no information
// to avoid exposure. The request should not proceed.
ctxu.GetLogger(context).Errorf("error checking authorization: %v", err)
w.WriteHeader(http.StatusBadRequest)
}
return err
}
// TODO(stevvooe): This pattern needs to be cleaned up a bit. One context
// should be replaced by another, rather than replacing the context on a
// mutable object.
context.Context = ctx
return nil
}
// eventBridge returns a bridge for the current request, configured with the
// correct actor and source.
func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listener {
actor := notifications.ActorRecord{
Name: getUserName(ctx, r),
}
request := notifications.NewRequestRecord(ctxu.GetRequestID(ctx), r)
return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink)
}
// nameRequired returns true if the route requires a name.
func (app *App) nameRequired(r *http.Request) bool {
route := mux.CurrentRoute(r)
routeName := route.GetName()
return route == nil || (routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog)
}
// apiBase implements a simple yes-man for doing overall checks against the
// api. This can support auth roundtrips to support docker login.
func apiBase(w http.ResponseWriter, r *http.Request) {
const emptyJSON = "{}"
// Provide a simple /v2/ 200 OK response with empty json response.
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprint(len(emptyJSON)))
fmt.Fprint(w, emptyJSON)
}
// appendAccessRecords checks the method and adds the appropriate Access records to the records list.
func appendAccessRecords(records []auth.Access, method string, repo string) []auth.Access {
resource := auth.Resource{
Type: "repository",
Name: repo,
}
switch method {
case "GET", "HEAD":
records = append(records,
auth.Access{
Resource: resource,
Action: "pull",
})
case "POST", "PUT", "PATCH":
records = append(records,
auth.Access{
Resource: resource,
Action: "pull",
},
auth.Access{
Resource: resource,
Action: "push",
})
case "DELETE":
// DELETE access requires full admin rights, which is represented
// as "*". This may not be ideal.
records = append(records,
auth.Access{
Resource: resource,
Action: "*",
})
}
return records
}
// Add the access record for the catalog if it's our current route
func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access {
route := mux.CurrentRoute(r)
routeName := route.GetName()
if routeName == v2.RouteNameCatalog {
resource := auth.Resource{
Type: "registry",
Name: "catalog",
}
accessRecords = append(accessRecords,
auth.Access{
Resource: resource,
Action: "*",
})
}
return accessRecords
}
// applyRegistryMiddleware wraps a registry instance with the configured middlewares
func applyRegistryMiddleware(ctx context.Context, registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) {
for _, mw := range middlewares {
rmw, err := registrymiddleware.Get(ctx, mw.Name, mw.Options, registry)
if err != nil {
return nil, fmt.Errorf("unable to configure registry middleware (%s): %s", mw.Name, err)
}
registry = rmw
}
return registry, nil
}
// applyRepoMiddleware wraps a repository with the configured middlewares
func applyRepoMiddleware(ctx context.Context, repository distribution.Repository, middlewares []configuration.Middleware) (distribution.Repository, error) {
for _, mw := range middlewares {
rmw, err := repositorymiddleware.Get(ctx, mw.Name, mw.Options, repository)
if err != nil {
return nil, err
}
repository = rmw
}
return repository, nil
}
// applyStorageMiddleware wraps a storage driver with the configured middlewares
func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []configuration.Middleware) (storagedriver.StorageDriver, error) {
for _, mw := range middlewares {
smw, err := storagemiddleware.Get(mw.Name, mw.Options, driver)
if err != nil {
return nil, fmt.Errorf("unable to configure storage middleware (%s): %v", mw.Name, err)
}
driver = smw
}
return driver, nil
}
// uploadPurgeDefaultConfig provides a default configuration for upload
// purging to be used in the absence of configuration in the
// confifuration file
func uploadPurgeDefaultConfig() map[interface{}]interface{} {
config := map[interface{}]interface{}{}
config["enabled"] = true
config["age"] = "168h"
config["interval"] = "24h"
config["dryrun"] = false
return config
}
func badPurgeUploadConfig(reason string) {
panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason))
}
// startUploadPurger schedules a goroutine which will periodically
// check upload directories for old files and delete them
func startUploadPurger(ctx context.Context, storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) {
if config["enabled"] == false {
return
}
var purgeAgeDuration time.Duration
var err error
purgeAge, ok := config["age"]
if ok {
ageStr, ok := purgeAge.(string)
if !ok {
badPurgeUploadConfig("age is not a string")
}
purgeAgeDuration, err = time.ParseDuration(ageStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error()))
}
} else {
badPurgeUploadConfig("age missing")
}
var intervalDuration time.Duration
interval, ok := config["interval"]
if ok {
intervalStr, ok := interval.(string)
if !ok {
badPurgeUploadConfig("interval is not a string")
}
intervalDuration, err = time.ParseDuration(intervalStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error()))
}
} else {
badPurgeUploadConfig("interval missing")
}
var dryRunBool bool
dryRun, ok := config["dryrun"]
if ok {
dryRunBool, ok = dryRun.(bool)
if !ok {
badPurgeUploadConfig("cannot parse dryrun")
}
} else {
badPurgeUploadConfig("dryrun missing")
}
go func() {
rand.Seed(time.Now().Unix())
jitter := time.Duration(rand.Int()%60) * time.Minute
log.Infof("Starting upload purge in %s", jitter)
time.Sleep(jitter)
for {
storage.PurgeUploads(ctx, storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool)
log.Infof("Starting upload purge in %s", intervalDuration)
time.Sleep(intervalDuration)
}
}()
}

274
docs/handlers/app_test.go
View file

@ -1,274 +0,0 @@
package handlers
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"testing"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/auth"
_ "github.com/docker/distribution/registry/auth/silly"
"github.com/docker/distribution/registry/storage"
memorycache "github.com/docker/distribution/registry/storage/cache/memory"
"github.com/docker/distribution/registry/storage/driver/testdriver"
)
// TestAppDispatcher builds an application with a test dispatcher and ensures
// that requests are properly dispatched and the handlers are constructed.
// This only tests the dispatch mechanism. The underlying dispatchers must be
// tested individually.
func TestAppDispatcher(t *testing.T) {
driver := testdriver.New()
ctx := context.Background()
registry, err := storage.NewRegistry(ctx, driver, storage.BlobDescriptorCacheProvider(memorycache.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
app := &App{
Config: &configuration.Configuration{},
Context: ctx,
router: v2.Router(),
driver: driver,
registry: registry,
}
server := httptest.NewServer(app)
router := v2.Router()
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("error parsing server url: %v", err)
}
varCheckingDispatcher := func(expectedVars map[string]string) dispatchFunc {
return func(ctx *Context, r *http.Request) http.Handler {
// Always checks the same name context
if ctx.Repository.Named().Name() != getName(ctx) {
t.Fatalf("unexpected name: %q != %q", ctx.Repository.Named().Name(), "foo/bar")
}
// Check that we have all that is expected
for expectedK, expectedV := range expectedVars {
if ctx.Value(expectedK) != expectedV {
t.Fatalf("unexpected %s in context vars: %q != %q", expectedK, ctx.Value(expectedK), expectedV)
}
}
// Check that we only have variables that are expected
for k, v := range ctx.Value("vars").(map[string]string) {
_, ok := expectedVars[k]
if !ok { // name is checked on context
// We have an unexpected key, fail
t.Fatalf("unexpected key %q in vars with value %q", k, v)
}
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
}
}
// unflatten a list of variables, suitable for gorilla/mux, to a map[string]string
unflatten := func(vars []string) map[string]string {
m := make(map[string]string)
for i := 0; i < len(vars)-1; i = i + 2 {
m[vars[i]] = vars[i+1]
}
return m
}
for _, testcase := range []struct {
endpoint string
vars []string
}{
{
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "foo/bar",
"reference", "sometag",
},
},
{
endpoint: v2.RouteNameTags,
vars: []string{
"name", "foo/bar",
},
},
{
endpoint: v2.RouteNameBlobUpload,
vars: []string{
"name", "foo/bar",
},
},
{
endpoint: v2.RouteNameBlobUploadChunk,
vars: []string{
"name", "foo/bar",
"uuid", "theuuid",
},
},
} {
app.register(testcase.endpoint, varCheckingDispatcher(unflatten(testcase.vars)))
route := router.GetRoute(testcase.endpoint).Host(serverURL.Host)
u, err := route.URL(testcase.vars...)
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(u.String())
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %v != %v", resp.StatusCode, http.StatusOK)
}
}
}
// TestNewApp covers the creation of an application via NewApp with a
// configuration.
func TestNewApp(t *testing.T) {
ctx := context.Background()
config := configuration.Configuration{
Storage: configuration.Storage{
"testdriver": nil,
},
Auth: configuration.Auth{
// For now, we simply test that new auth results in a viable
// application.
"silly": {
"realm": "realm-test",
"service": "service-test",
},
},
}
// Mostly, with this test, given a sane configuration, we are simply
// ensuring that NewApp doesn't panic. We might want to tweak this
// behavior.
app := NewApp(ctx, &config)
server := httptest.NewServer(app)
builder, err := v2.NewURLBuilderFromString(server.URL, false)
if err != nil {
t.Fatalf("error creating urlbuilder: %v", err)
}
baseURL, err := builder.BuildBaseURL()
if err != nil {
t.Fatalf("error creating baseURL: %v", err)
}
// TODO(stevvooe): The rest of this test might belong in the API tests.
// Just hit the app and make sure we get a 401 Unauthorized error.
req, err := http.Get(baseURL)
if err != nil {
t.Fatalf("unexpected error during GET: %v", err)
}
defer req.Body.Close()
if req.StatusCode != http.StatusUnauthorized {
t.Fatalf("unexpected status code during request: %v", err)
}
if req.Header.Get("Content-Type") != "application/json; charset=utf-8" {
t.Fatalf("unexpected content-type: %v != %v", req.Header.Get("Content-Type"), "application/json; charset=utf-8")
}
expectedAuthHeader := "Bearer realm=\"realm-test\",service=\"service-test\""
if e, a := expectedAuthHeader, req.Header.Get("WWW-Authenticate"); e != a {
t.Fatalf("unexpected WWW-Authenticate header: %q != %q", e, a)
}
var errs errcode.Errors
dec := json.NewDecoder(req.Body)
if err := dec.Decode(&errs); err != nil {
t.Fatalf("error decoding error response: %v", err)
}
err2, ok := errs[0].(errcode.ErrorCoder)
if !ok {
t.Fatalf("not an ErrorCoder: %#v", errs[0])
}
if err2.ErrorCode() != errcode.ErrorCodeUnauthorized {
t.Fatalf("unexpected error code: %v != %v", err2.ErrorCode(), errcode.ErrorCodeUnauthorized)
}
}
// Test the access record accumulator
func TestAppendAccessRecords(t *testing.T) {
repo := "testRepo"
expectedResource := auth.Resource{
Type: "repository",
Name: repo,
}
expectedPullRecord := auth.Access{
Resource: expectedResource,
Action: "pull",
}
expectedPushRecord := auth.Access{
Resource: expectedResource,
Action: "push",
}
expectedAllRecord := auth.Access{
Resource: expectedResource,
Action: "*",
}
records := []auth.Access{}
result := appendAccessRecords(records, "GET", repo)
expectedResult := []auth.Access{expectedPullRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
records = []auth.Access{}
result = appendAccessRecords(records, "HEAD", repo)
expectedResult = []auth.Access{expectedPullRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
records = []auth.Access{}
result = appendAccessRecords(records, "POST", repo)
expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
records = []auth.Access{}
result = appendAccessRecords(records, "PUT", repo)
expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
records = []auth.Access{}
result = appendAccessRecords(records, "PATCH", repo)
expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
records = []auth.Access{}
result = appendAccessRecords(records, "DELETE", repo)
expectedResult = []auth.Access{expectedAllRecord}
if ok := reflect.DeepEqual(result, expectedResult); !ok {
t.Fatalf("Actual access record differs from expected")
}
}

11
docs/handlers/basicauth.go
View file

@ -1,11 +0,0 @@
// +build go1.4
package handlers
import (
"net/http"
)
func basicAuth(r *http.Request) (username, password string, ok bool) {
return r.BasicAuth()
}

41
docs/handlers/basicauth_prego14.go
View file

@ -1,41 +0,0 @@
// +build !go1.4
package handlers
import (
"encoding/base64"
"net/http"
"strings"
)
// NOTE(stevvooe): This is basic auth support from go1.4 present to ensure we
// can compile on go1.3 and earlier.
// BasicAuth returns the username and password provided in the request's
// Authorization header, if the request uses HTTP Basic Authentication.
// See RFC 2617, Section 2.
func basicAuth(r *http.Request) (username, password string, ok bool) {
auth := r.Header.Get("Authorization")
if auth == "" {
return
}
return parseBasicAuth(auth)
}
// parseBasicAuth parses an HTTP Basic Authentication string.
// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true).
func parseBasicAuth(auth string) (username, password string, ok bool) {
if !strings.HasPrefix(auth, "Basic ") {
return
}
c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic "))
if err != nil {
return
}
cs := string(c)
s := strings.IndexByte(cs, ':')
if s < 0 {
return
}
return cs[:s], cs[s+1:], true
}

99
docs/handlers/blob.go
View file

@ -1,99 +0,0 @@
package handlers
import (
"net/http"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
)
// blobDispatcher uses the request context to build a blobHandler.
func blobDispatcher(ctx *Context, r *http.Request) http.Handler {
dgst, err := getDigest(ctx)
if err != nil {
if err == errDigestNotAvailable {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx.Errors = append(ctx.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
})
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx.Errors = append(ctx.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
})
}
blobHandler := &blobHandler{
Context: ctx,
Digest: dgst,
}
mhandler := handlers.MethodHandler{
"GET": http.HandlerFunc(blobHandler.GetBlob),
"HEAD": http.HandlerFunc(blobHandler.GetBlob),
}
if !ctx.readOnly {
mhandler["DELETE"] = http.HandlerFunc(blobHandler.DeleteBlob)
}
return mhandler
}
// blobHandler serves http blob requests.
type blobHandler struct {
*Context
Digest digest.Digest
}
// GetBlob fetches the binary data from backend storage returns it in the
// response.
func (bh *blobHandler) GetBlob(w http.ResponseWriter, r *http.Request) {
context.GetLogger(bh).Debug("GetBlob")
blobs := bh.Repository.Blobs(bh)
desc, err := blobs.Stat(bh, bh.Digest)
if err != nil {
if err == distribution.ErrBlobUnknown {
bh.Errors = append(bh.Errors, v2.ErrorCodeBlobUnknown.WithDetail(bh.Digest))
} else {
bh.Errors = append(bh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
if err := blobs.ServeBlob(bh, w, r, desc.Digest); err != nil {
context.GetLogger(bh).Debugf("unexpected error getting blob HTTP handler: %v", err)
bh.Errors = append(bh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// DeleteBlob deletes a layer blob
func (bh *blobHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) {
context.GetLogger(bh).Debug("DeleteBlob")
blobs := bh.Repository.Blobs(bh)
err := blobs.Delete(bh, bh.Digest)
if err != nil {
switch err {
case distribution.ErrUnsupported:
bh.Errors = append(bh.Errors, errcode.ErrorCodeUnsupported)
return
case distribution.ErrBlobUnknown:
bh.Errors = append(bh.Errors, v2.ErrorCodeBlobUnknown)
return
default:
bh.Errors = append(bh.Errors, err)
context.GetLogger(bh).Errorf("Unknown error deleting blob: %s", err.Error())
return
}
}
w.Header().Set("Content-Length", "0")
w.WriteHeader(http.StatusAccepted)
}

368
docs/handlers/blobupload.go
View file

@ -1,368 +0,0 @@
package handlers
import (
"fmt"
"net/http"
"net/url"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/storage"
"github.com/gorilla/handlers"
)
// blobUploadDispatcher constructs and returns the blob upload handler for the
// given request context.
func blobUploadDispatcher(ctx *Context, r *http.Request) http.Handler {
buh := &blobUploadHandler{
Context: ctx,
UUID: getUploadUUID(ctx),
}
handler := handlers.MethodHandler{
"GET": http.HandlerFunc(buh.GetUploadStatus),
"HEAD": http.HandlerFunc(buh.GetUploadStatus),
}
if !ctx.readOnly {
handler["POST"] = http.HandlerFunc(buh.StartBlobUpload)
handler["PATCH"] = http.HandlerFunc(buh.PatchBlobData)
handler["PUT"] = http.HandlerFunc(buh.PutBlobUploadComplete)
handler["DELETE"] = http.HandlerFunc(buh.CancelBlobUpload)
}
if buh.UUID != "" {
state, err := hmacKey(ctx.Config.HTTP.Secret).unpackUploadState(r.FormValue("_state"))
if err != nil {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(ctx).Infof("error resolving upload: %v", err)
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err))
})
}
buh.State = state
if state.Name != ctx.Repository.Named().Name() {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(ctx).Infof("mismatched repository name in upload state: %q != %q", state.Name, buh.Repository.Named().Name())
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err))
})
}
if state.UUID != buh.UUID {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(ctx).Infof("mismatched uuid in upload state: %q != %q", state.UUID, buh.UUID)
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err))
})
}
blobs := ctx.Repository.Blobs(buh)
upload, err := blobs.Resume(buh, buh.UUID)
if err != nil {
ctxu.GetLogger(ctx).Errorf("error resolving upload: %v", err)
if err == distribution.ErrBlobUploadUnknown {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown.WithDetail(err))
})
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
})
}
buh.Upload = upload
if size := upload.Size(); size != buh.State.Offset {
defer upload.Close()
ctxu.GetLogger(ctx).Errorf("upload resumed at wrong offest: %d != %d", size, buh.State.Offset)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err))
upload.Cancel(buh)
})
}
return closeResources(handler, buh.Upload)
}
return handler
}
// blobUploadHandler handles the http blob upload process.
type blobUploadHandler struct {
*Context
// UUID identifies the upload instance for the current request. Using UUID
// to key blob writers since this implementation uses UUIDs.
UUID string
Upload distribution.BlobWriter
State blobUploadState
}
// StartBlobUpload begins the blob upload process and allocates a server-side
// blob writer session, optionally mounting the blob from a separate repository.
func (buh *blobUploadHandler) StartBlobUpload(w http.ResponseWriter, r *http.Request) {
var options []distribution.BlobCreateOption
fromRepo := r.FormValue("from")
mountDigest := r.FormValue("mount")
if mountDigest != "" && fromRepo != "" {
opt, err := buh.createBlobMountOption(fromRepo, mountDigest)
if opt != nil && err == nil {
options = append(options, opt)
}
}
blobs := buh.Repository.Blobs(buh)
upload, err := blobs.Create(buh, options...)
if err != nil {
if ebm, ok := err.(distribution.ErrBlobMounted); ok {
if err := buh.writeBlobCreatedHeaders(w, ebm.Descriptor); err != nil {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
} else if err == distribution.ErrUnsupported {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnsupported)
} else {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
buh.Upload = upload
if err := buh.blobUploadResponse(w, r, true); err != nil {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Docker-Upload-UUID", buh.Upload.ID())
w.WriteHeader(http.StatusAccepted)
}
// GetUploadStatus returns the status of a given upload, identified by id.
func (buh *blobUploadHandler) GetUploadStatus(w http.ResponseWriter, r *http.Request) {
if buh.Upload == nil {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown)
return
}
// TODO(dmcgowan): Set last argument to false in blobUploadResponse when
// resumable upload is supported. This will enable returning a non-zero
// range for clients to begin uploading at an offset.
if err := buh.blobUploadResponse(w, r, true); err != nil {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Docker-Upload-UUID", buh.UUID)
w.WriteHeader(http.StatusNoContent)
}
// PatchBlobData writes data to an upload.
func (buh *blobUploadHandler) PatchBlobData(w http.ResponseWriter, r *http.Request) {
if buh.Upload == nil {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown)
return
}
ct := r.Header.Get("Content-Type")
if ct != "" && ct != "application/octet-stream" {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(fmt.Errorf("Bad Content-Type")))
// TODO(dmcgowan): encode error
return
}
// TODO(dmcgowan): support Content-Range header to seek and write range
if err := copyFullPayload(w, r, buh.Upload, buh, "blob PATCH", &buh.Errors); err != nil {
// copyFullPayload reports the error if necessary
return
}
if err := buh.blobUploadResponse(w, r, false); err != nil {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.WriteHeader(http.StatusAccepted)
}
// PutBlobUploadComplete takes the final request of a blob upload. The
// request may include all the blob data or no blob data. Any data
// provided is received and verified. If successful, the blob is linked
// into the blob store and 201 Created is returned with the canonical
// url of the blob.
func (buh *blobUploadHandler) PutBlobUploadComplete(w http.ResponseWriter, r *http.Request) {
if buh.Upload == nil {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown)
return
}
dgstStr := r.FormValue("digest") // TODO(stevvooe): Support multiple digest parameters!
if dgstStr == "" {
// no digest? return error, but allow retry.
buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail("digest missing"))
return
}
dgst, err := digest.ParseDigest(dgstStr)
if err != nil {
// no digest? return error, but allow retry.
buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail("digest parsing failed"))
return
}
if err := copyFullPayload(w, r, buh.Upload, buh, "blob PUT", &buh.Errors); err != nil {
// copyFullPayload reports the error if necessary
return
}
desc, err := buh.Upload.Commit(buh, distribution.Descriptor{
Digest: dgst,
// TODO(stevvooe): This isn't wildly important yet, but we should
// really set the mediatype. For now, we can let the backend take care
// of this.
})
if err != nil {
switch err := err.(type) {
case distribution.ErrBlobInvalidDigest:
buh.Errors = append(buh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
case errcode.Error:
buh.Errors = append(buh.Errors, err)
default:
switch err {
case distribution.ErrAccessDenied:
buh.Errors = append(buh.Errors, errcode.ErrorCodeDenied)
case distribution.ErrUnsupported:
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnsupported)
case distribution.ErrBlobInvalidLength, distribution.ErrBlobDigestUnsupported:
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadInvalid.WithDetail(err))
default:
ctxu.GetLogger(buh).Errorf("unknown error completing upload: %v", err)
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
}
// Clean up the backend blob data if there was an error.
if err := buh.Upload.Cancel(buh); err != nil {
// If the cleanup fails, all we can do is observe and report.
ctxu.GetLogger(buh).Errorf("error canceling upload after error: %v", err)
}
return
}
if err := buh.writeBlobCreatedHeaders(w, desc); err != nil {
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// CancelBlobUpload cancels an in-progress upload of a blob.
func (buh *blobUploadHandler) CancelBlobUpload(w http.ResponseWriter, r *http.Request) {
if buh.Upload == nil {
buh.Errors = append(buh.Errors, v2.ErrorCodeBlobUploadUnknown)
return
}
w.Header().Set("Docker-Upload-UUID", buh.UUID)
if err := buh.Upload.Cancel(buh); err != nil {
ctxu.GetLogger(buh).Errorf("error encountered canceling upload: %v", err)
buh.Errors = append(buh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
w.WriteHeader(http.StatusNoContent)
}
// blobUploadResponse provides a standard request for uploading blobs and
// chunk responses. This sets the correct headers but the response status is
// left to the caller. The fresh argument is used to ensure that new blob
// uploads always start at a 0 offset. This allows disabling resumable push by
// always returning a 0 offset on check status.
func (buh *blobUploadHandler) blobUploadResponse(w http.ResponseWriter, r *http.Request, fresh bool) error {
// TODO(stevvooe): Need a better way to manage the upload state automatically.
buh.State.Name = buh.Repository.Named().Name()
buh.State.UUID = buh.Upload.ID()
buh.Upload.Close()
buh.State.Offset = buh.Upload.Size()
buh.State.StartedAt = buh.Upload.StartedAt()
token, err := hmacKey(buh.Config.HTTP.Secret).packUploadState(buh.State)
if err != nil {
ctxu.GetLogger(buh).Infof("error building upload state token: %s", err)
return err
}
uploadURL, err := buh.urlBuilder.BuildBlobUploadChunkURL(
buh.Repository.Named(), buh.Upload.ID(),
url.Values{
"_state": []string{token},
})
if err != nil {
ctxu.GetLogger(buh).Infof("error building upload url: %s", err)
return err
}
endRange := buh.Upload.Size()
if endRange > 0 {
endRange = endRange - 1
}
w.Header().Set("Docker-Upload-UUID", buh.UUID)
w.Header().Set("Location", uploadURL)
w.Header().Set("Content-Length", "0")
w.Header().Set("Range", fmt.Sprintf("0-%d", endRange))
return nil
}
// mountBlob attempts to mount a blob from another repository by its digest. If
// successful, the blob is linked into the blob store and 201 Created is
// returned with the canonical url of the blob.
func (buh *blobUploadHandler) createBlobMountOption(fromRepo, mountDigest string) (distribution.BlobCreateOption, error) {
dgst, err := digest.ParseDigest(mountDigest)
if err != nil {
return nil, err
}
ref, err := reference.ParseNamed(fromRepo)
if err != nil {
return nil, err
}
canonical, err := reference.WithDigest(ref, dgst)
if err != nil {
return nil, err
}
return storage.WithMountFrom(canonical), nil
}
// writeBlobCreatedHeaders writes the standard headers describing a newly
// created blob. A 201 Created is written as well as the canonical URL and
// blob digest.
func (buh *blobUploadHandler) writeBlobCreatedHeaders(w http.ResponseWriter, desc distribution.Descriptor) error {
ref, err := reference.WithDigest(buh.Repository.Named(), desc.Digest)
if err != nil {
return err
}
blobURL, err := buh.urlBuilder.BuildBlobURL(ref)
if err != nil {
return err
}
w.Header().Set("Location", blobURL)
w.Header().Set("Content-Length", "0")
w.Header().Set("Docker-Content-Digest", desc.Digest.String())
w.WriteHeader(http.StatusCreated)
return nil
}

95
docs/handlers/catalog.go
View file

@ -1,95 +0,0 @@
package handlers
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"github.com/docker/distribution/registry/api/errcode"
"github.com/gorilla/handlers"
)
const maximumReturnedEntries = 100
func catalogDispatcher(ctx *Context, r *http.Request) http.Handler {
catalogHandler := &catalogHandler{
Context: ctx,
}
return handlers.MethodHandler{
"GET": http.HandlerFunc(catalogHandler.GetCatalog),
}
}
type catalogHandler struct {
*Context
}
type catalogAPIResponse struct {
Repositories []string `json:"repositories"`
}
func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) {
var moreEntries = true
q := r.URL.Query()
lastEntry := q.Get("last")
maxEntries, err := strconv.Atoi(q.Get("n"))
if err != nil || maxEntries < 0 {
maxEntries = maximumReturnedEntries
}
repos := make([]string, maxEntries)
filled, err := ch.App.registry.Repositories(ch.Context, repos, lastEntry)
if err == io.EOF {
moreEntries = false
} else if err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
// Add a link header if there are more entries to retrieve
if moreEntries {
lastEntry = repos[len(repos)-1]
urlStr, err := createLinkEntry(r.URL.String(), maxEntries, lastEntry)
if err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Link", urlStr)
}
enc := json.NewEncoder(w)
if err := enc.Encode(catalogAPIResponse{
Repositories: repos[0:filled],
}); err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Use the original URL from the request to create a new URL for
// the link header
func createLinkEntry(origURL string, maxEntries int, lastEntry string) (string, error) {
calledURL, err := url.Parse(origURL)
if err != nil {
return "", err
}
v := url.Values{}
v.Add("n", strconv.Itoa(maxEntries))
v.Add("last", lastEntry)
calledURL.RawQuery = v.Encode()
calledURL.Fragment = ""
urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String())
return urlStr, nil
}

152
docs/handlers/context.go
View file

@ -1,152 +0,0 @@
package handlers
import (
"fmt"
"net/http"
"sync"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/auth"
"golang.org/x/net/context"
)
// Context should contain the request specific context for use in across
// handlers. Resources that don't need to be shared across handlers should not
// be on this object.
type Context struct {
// App points to the application structure that created this context.
*App
context.Context
// Repository is the repository for the current request. All requests
// should be scoped to a single repository. This field may be nil.
Repository distribution.Repository
// Errors is a collection of errors encountered during the request to be
// returned to the client API. If errors are added to the collection, the
// handler *must not* start the response via http.ResponseWriter.
Errors errcode.Errors
urlBuilder *v2.URLBuilder
// TODO(stevvooe): The goal is too completely factor this context and
// dispatching out of the web application. Ideally, we should lean on
// context.Context for injection of these resources.
}
// Value overrides context.Context.Value to ensure that calls are routed to
// correct context.
func (ctx *Context) Value(key interface{}) interface{} {
return ctx.Context.Value(key)
}
func getName(ctx context.Context) (name string) {
return ctxu.GetStringValue(ctx, "vars.name")
}
func getReference(ctx context.Context) (reference string) {
return ctxu.GetStringValue(ctx, "vars.reference")
}
var errDigestNotAvailable = fmt.Errorf("digest not available in context")
func getDigest(ctx context.Context) (dgst digest.Digest, err error) {
dgstStr := ctxu.GetStringValue(ctx, "vars.digest")
if dgstStr == "" {
ctxu.GetLogger(ctx).Errorf("digest not available")
return "", errDigestNotAvailable
}
d, err := digest.ParseDigest(dgstStr)
if err != nil {
ctxu.GetLogger(ctx).Errorf("error parsing digest=%q: %v", dgstStr, err)
return "", err
}
return d, nil
}
func getUploadUUID(ctx context.Context) (uuid string) {
return ctxu.GetStringValue(ctx, "vars.uuid")
}
// getUserName attempts to resolve a username from the context and request. If
// a username cannot be resolved, the empty string is returned.
func getUserName(ctx context.Context, r *http.Request) string {
username := ctxu.GetStringValue(ctx, auth.UserNameKey)
// Fallback to request user with basic auth
if username == "" {
var ok bool
uname, _, ok := basicAuth(r)
if ok {
username = uname
}
}
return username
}
// contextManager allows us to associate net/context.Context instances with a
// request, based on the memory identity of http.Request. This prepares http-
// level context, which is not application specific. If this is called,
// (*contextManager).release must be called on the context when the request is
// completed.
//
// Providing this circumvents a lot of necessity for dispatchers with the
// benefit of instantiating the request context much earlier.
//
// TODO(stevvooe): Consider making this facility a part of the context package.
type contextManager struct {
contexts map[*http.Request]context.Context
mu sync.Mutex
}
// defaultContextManager is just a global instance to register request contexts.
var defaultContextManager = newContextManager()
func newContextManager() *contextManager {
return &contextManager{
contexts: make(map[*http.Request]context.Context),
}
}
// context either returns a new context or looks it up in the manager.
func (cm *contextManager) context(parent context.Context, w http.ResponseWriter, r *http.Request) context.Context {
cm.mu.Lock()
defer cm.mu.Unlock()
ctx, ok := cm.contexts[r]
if ok {
return ctx
}
if parent == nil {
parent = ctxu.Background()
}
ctx = ctxu.WithRequest(parent, r)
ctx, w = ctxu.WithResponseWriter(ctx, w)
ctx = ctxu.WithLogger(ctx, ctxu.GetRequestLogger(ctx))
cm.contexts[r] = ctx
return ctx
}
// releases frees any associated with resources from request.
func (cm *contextManager) release(ctx context.Context) {
cm.mu.Lock()
defer cm.mu.Unlock()
r, err := ctxu.GetRequest(ctx)
if err != nil {
ctxu.GetLogger(ctx).Errorf("no request found in context during release")
return
}
delete(cm.contexts, r)
}

201
docs/handlers/health_test.go
View file

@ -1,201 +0,0 @@
package handlers
import (
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/context"
"github.com/docker/distribution/health"
)
func TestFileHealthCheck(t *testing.T) {
interval := time.Second
tmpfile, err := ioutil.TempFile(os.TempDir(), "healthcheck")
if err != nil {
t.Fatalf("could not create temporary file: %v", err)
}
defer tmpfile.Close()
config := &configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Health: configuration.Health{
FileCheckers: []configuration.FileChecker{
{
Interval: interval,
File: tmpfile.Name(),
},
},
},
}
ctx := context.Background()
app := NewApp(ctx, config)
healthRegistry := health.NewRegistry()
app.RegisterHealthChecks(healthRegistry)
// Wait for health check to happen
<-time.After(2 * interval)
status := healthRegistry.CheckStatus()
if len(status) != 1 {
t.Fatal("expected 1 item in health check results")
}
if status[tmpfile.Name()] != "file exists" {
t.Fatal(`did not get "file exists" result for health check`)
}
os.Remove(tmpfile.Name())
<-time.After(2 * interval)
if len(healthRegistry.CheckStatus()) != 0 {
t.Fatal("expected 0 items in health check results")
}
}
func TestTCPHealthCheck(t *testing.T) {
interval := time.Second
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("could not create listener: %v", err)
}
addrStr := ln.Addr().String()
// Start accepting
go func() {
for {
conn, err := ln.Accept()
if err != nil {
// listener was closed
return
}
defer conn.Close()
}
}()
config := &configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Health: configuration.Health{
TCPCheckers: []configuration.TCPChecker{
{
Interval: interval,
Addr: addrStr,
Timeout: 500 * time.Millisecond,
},
},
},
}
ctx := context.Background()
app := NewApp(ctx, config)
healthRegistry := health.NewRegistry()
app.RegisterHealthChecks(healthRegistry)
// Wait for health check to happen
<-time.After(2 * interval)
if len(healthRegistry.CheckStatus()) != 0 {
t.Fatal("expected 0 items in health check results")
}
ln.Close()
<-time.After(2 * interval)
// Health check should now fail
status := healthRegistry.CheckStatus()
if len(status) != 1 {
t.Fatal("expected 1 item in health check results")
}
if status[addrStr] != "connection to "+addrStr+" failed" {
t.Fatal(`did not get "connection failed" result for health check`)
}
}
func TestHTTPHealthCheck(t *testing.T) {
interval := time.Second
threshold := 3
stopFailing := make(chan struct{})
checkedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "HEAD" {
t.Fatalf("expected HEAD request, got %s", r.Method)
}
select {
case <-stopFailing:
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusInternalServerError)
}
}))
config := &configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Health: configuration.Health{
HTTPCheckers: []configuration.HTTPChecker{
{
Interval: interval,
URI: checkedServer.URL,
Threshold: threshold,
},
},
},
}
ctx := context.Background()
app := NewApp(ctx, config)
healthRegistry := health.NewRegistry()
app.RegisterHealthChecks(healthRegistry)
for i := 0; ; i++ {
<-time.After(interval)
status := healthRegistry.CheckStatus()
if i < threshold-1 {
// definitely shouldn't have hit the threshold yet
if len(status) != 0 {
t.Fatal("expected 1 item in health check results")
}
continue
}
if i < threshold+1 {
// right on the threshold - don't expect a failure yet
continue
}
if len(status) != 1 {
t.Fatal("expected 1 item in health check results")
}
if status[checkedServer.URL] != "downstream service returned unexpected status: 500" {
t.Fatal("did not get expected result for health check")
}
break
}
// Signal HTTP handler to start returning 200
close(stopFailing)
<-time.After(2 * interval)
if len(healthRegistry.CheckStatus()) != 0 {
t.Fatal("expected 0 items in health check results")
}
}

66
docs/handlers/helpers.go
View file

@ -1,66 +0,0 @@
package handlers
import (
"errors"
"io"
"net/http"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/registry/api/errcode"
)
// closeResources closes all the provided resources after running the target
// handler.
func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for _, closer := range closers {
defer closer.Close()
}
handler.ServeHTTP(w, r)
})
}
// copyFullPayload copies the payload of an HTTP request to destWriter. If it
// receives less content than expected, and the client disconnected during the
// upload, it avoids sending a 400 error to keep the logs cleaner.
func copyFullPayload(responseWriter http.ResponseWriter, r *http.Request, destWriter io.Writer, context ctxu.Context, action string, errSlice *errcode.Errors) error {
// Get a channel that tells us if the client disconnects
var clientClosed <-chan bool
if notifier, ok := responseWriter.(http.CloseNotifier); ok {
clientClosed = notifier.CloseNotify()
} else {
ctxu.GetLogger(context).Warnf("the ResponseWriter does not implement CloseNotifier (type: %T)", responseWriter)
}
// Read in the data, if any.
copied, err := io.Copy(destWriter, r.Body)
if clientClosed != nil && (err != nil || (r.ContentLength > 0 && copied < r.ContentLength)) {
// Didn't receive as much content as expected. Did the client
// disconnect during the request? If so, avoid returning a 400
// error to keep the logs cleaner.
select {
case <-clientClosed:
// Set the response code to "499 Client Closed Request"
// Even though the connection has already been closed,
// this causes the logger to pick up a 499 error
// instead of showing 0 for the HTTP status.
responseWriter.WriteHeader(499)
ctxu.GetLoggerWithFields(context, map[interface{}]interface{}{
"error": err,
"copied": copied,
"contentLength": r.ContentLength,
}, "error", "copied", "contentLength").Error("client disconnected during " + action)
return errors.New("client disconnected")
default:
}
}
if err != nil {
ctxu.GetLogger(context).Errorf("unknown error reading request payload: %v", err)
*errSlice = append(*errSlice, errcode.ErrorCodeUnknown.WithDetail(err))
return err
}
return nil
}

72
docs/handlers/hmac.go
View file

@ -1,72 +0,0 @@
package handlers
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"time"
)
// blobUploadState captures the state serializable state of the blob upload.
type blobUploadState struct {
// name is the primary repository under which the blob will be linked.
Name string
// UUID identifies the upload.
UUID string
// offset contains the current progress of the upload.
Offset int64
// StartedAt is the original start time of the upload.
StartedAt time.Time
}
type hmacKey string
// unpackUploadState unpacks and validates the blob upload state from the
// token, using the hmacKey secret.
func (secret hmacKey) unpackUploadState(token string) (blobUploadState, error) {
var state blobUploadState
tokenBytes, err := base64.URLEncoding.DecodeString(token)
if err != nil {
return state, err
}
mac := hmac.New(sha256.New, []byte(secret))
if len(tokenBytes) < mac.Size() {
return state, fmt.Errorf("Invalid token")
}
macBytes := tokenBytes[:mac.Size()]
messageBytes := tokenBytes[mac.Size():]
mac.Write(messageBytes)
if !hmac.Equal(mac.Sum(nil), macBytes) {
return state, fmt.Errorf("Invalid token")
}
if err := json.Unmarshal(messageBytes, &state); err != nil {
return state, err
}
return state, nil
}
// packUploadState packs the upload state signed with and hmac digest using
// the hmacKey secret, encoding to url safe base64. The resulting token can be
// used to share data with minimized risk of external tampering.
func (secret hmacKey) packUploadState(lus blobUploadState) (string, error) {
mac := hmac.New(sha256.New, []byte(secret))
p, err := json.Marshal(lus)
if err != nil {
return "", err
}
mac.Write(p)
return base64.URLEncoding.EncodeToString(append(mac.Sum(nil), p...)), nil
}

117
docs/handlers/hmac_test.go
View file

@ -1,117 +0,0 @@
package handlers
import "testing"
var blobUploadStates = []blobUploadState{
{
Name: "hello",
UUID: "abcd-1234-qwer-0987",
Offset: 0,
},
{
Name: "hello-world",
UUID: "abcd-1234-qwer-0987",
Offset: 0,
},
{
Name: "h3ll0_w0rld",
UUID: "abcd-1234-qwer-0987",
Offset: 1337,
},
{
Name: "ABCDEFG",
UUID: "ABCD-1234-QWER-0987",
Offset: 1234567890,
},
{
Name: "this-is-A-sort-of-Long-name-for-Testing",
UUID: "dead-1234-beef-0987",
Offset: 8675309,
},
}
var secrets = []string{
"supersecret",
"12345",
"a",
"SuperSecret",
"Sup3r... S3cr3t!",
"This is a reasonably long secret key that is used for the purpose of testing.",
"\u2603+\u2744", // snowman+snowflake
}
// TestLayerUploadTokens constructs stateTokens from LayerUploadStates and
// validates that the tokens can be used to reconstruct the proper upload state.
func TestLayerUploadTokens(t *testing.T) {
secret := hmacKey("supersecret")
for _, testcase := range blobUploadStates {
token, err := secret.packUploadState(testcase)
if err != nil {
t.Fatal(err)
}
lus, err := secret.unpackUploadState(token)
if err != nil {
t.Fatal(err)
}
assertBlobUploadStateEquals(t, testcase, lus)
}
}
// TestHMACValidate ensures that any HMAC token providers are compatible if and
// only if they share the same secret.
func TestHMACValidation(t *testing.T) {
for _, secret := range secrets {
secret1 := hmacKey(secret)
secret2 := hmacKey(secret)
badSecret := hmacKey("DifferentSecret")
for _, testcase := range blobUploadStates {
token, err := secret1.packUploadState(testcase)
if err != nil {
t.Fatal(err)
}
lus, err := secret2.unpackUploadState(token)
if err != nil {
t.Fatal(err)
}
assertBlobUploadStateEquals(t, testcase, lus)
_, err = badSecret.unpackUploadState(token)
if err == nil {
t.Fatalf("Expected token provider to fail at retrieving state from token: %s", token)
}
badToken, err := badSecret.packUploadState(lus)
if err != nil {
t.Fatal(err)
}
_, err = secret1.unpackUploadState(badToken)
if err == nil {
t.Fatalf("Expected token provider to fail at retrieving state from token: %s", badToken)
}
_, err = secret2.unpackUploadState(badToken)
if err == nil {
t.Fatalf("Expected token provider to fail at retrieving state from token: %s", badToken)
}
}
}
}
func assertBlobUploadStateEquals(t *testing.T, expected blobUploadState, received blobUploadState) {
if expected.Name != received.Name {
t.Fatalf("Expected Name=%q, Received Name=%q", expected.Name, received.Name)
}
if expected.UUID != received.UUID {
t.Fatalf("Expected UUID=%q, Received UUID=%q", expected.UUID, received.UUID)
}
if expected.Offset != received.Offset {
t.Fatalf("Expected Offset=%d, Received Offset=%d", expected.Offset, received.Offset)
}
}

53
docs/handlers/hooks.go
View file

@ -1,53 +0,0 @@
package handlers
import (
"bytes"
"errors"
"fmt"
"strings"
"text/template"
"github.com/Sirupsen/logrus"
)
// logHook is for hooking Panic in web application
type logHook struct {
LevelsParam []string
Mail *mailer
}
// Fire forwards an error to LogHook
func (hook *logHook) Fire(entry *logrus.Entry) error {
addr := strings.Split(hook.Mail.Addr, ":")
if len(addr) != 2 {
return errors.New("Invalid Mail Address")
}
host := addr[0]
subject := fmt.Sprintf("[%s] %s: %s", entry.Level, host, entry.Message)
html := `
{{.Message}}
{{range $key, $value := .Data}}
{{$key}}: {{$value}}
{{end}}
`
b := bytes.NewBuffer(make([]byte, 0))
t := template.Must(template.New("mail body").Parse(html))
if err := t.Execute(b, entry); err != nil {
return err
}
body := fmt.Sprintf("%s", b)
return hook.Mail.sendMail(subject, body)
}
// Levels contains hook levels to be catched
func (hook *logHook) Levels() []logrus.Level {
levels := []logrus.Level{}
for _, v := range hook.LevelsParam {
lv, _ := logrus.ParseLevel(v)
levels = append(levels, lv)
}
return levels
}

386
docs/handlers/images.go
View file

@ -1,386 +0,0 @@
package handlers
import (
"bytes"
"fmt"
"net/http"
"strings"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
)
// These constants determine which architecture and OS to choose from a
// manifest list when downconverting it to a schema1 manifest.
const (
defaultArch = "amd64"
defaultOS = "linux"
)
// imageManifestDispatcher takes the request context and builds the
// appropriate handler for handling image manifest requests.
func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler {
imageManifestHandler := &imageManifestHandler{
Context: ctx,
}
reference := getReference(ctx)
dgst, err := digest.ParseDigest(reference)
if err != nil {
// We just have a tag
imageManifestHandler.Tag = reference
} else {
imageManifestHandler.Digest = dgst
}
mhandler := handlers.MethodHandler{
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest),
}
if !ctx.readOnly {
mhandler["PUT"] = http.HandlerFunc(imageManifestHandler.PutImageManifest)
mhandler["DELETE"] = http.HandlerFunc(imageManifestHandler.DeleteImageManifest)
}
return mhandler
}
// imageManifestHandler handles http operations on image manifests.
type imageManifestHandler struct {
*Context
// One of tag or digest gets set, depending on what is present in context.
Tag string
Digest digest.Digest
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
supportsSchema2 := false
supportsManifestList := false
// this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values
// https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202
for _, acceptHeader := range r.Header["Accept"] {
// r.Header[...] is a slice in case the request contains the same header more than once
// if the header isn't set, we'll get the zero value, which "range" will handle gracefully
// we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616)
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
for _, mediaType := range strings.Split(acceptHeader, ",") {
// remove "; q=..." if present
if i := strings.Index(mediaType, ";"); i >= 0 {
mediaType = mediaType[:i]
}
// it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f")
mediaType = strings.TrimSpace(mediaType)
if mediaType == schema2.MediaTypeManifest {
supportsSchema2 = true
}
if mediaType == manifestlist.MediaTypeManifestList {
supportsManifestList = true
}
}
}
schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest)
manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList)
// Only rewrite schema2 manifests when they are being fetched by tag.
// If they are being fetched by digest, we can't return something not
// matching the digest.
if imh.Tag != "" && isSchema2 && !supportsSchema2 {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String())
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
} else if imh.Tag != "" && isManifestList && !supportsManifestList {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String())
// Find the image manifest corresponding to the default
// platform
var manifestDigest digest.Digest
for _, manifestDescriptor := range manifestList.Manifests {
if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS {
manifestDigest = manifestDescriptor.Digest
break
}
}
if manifestDigest == "" {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown)
return
}
manifest, err = manifests.Get(imh, manifestDigest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// If necessary, convert the image manifest
if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supportsSchema2 {
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
}
}
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(p)
}
func (imh *imageManifestHandler) convertSchema2Manifest(schema2Manifest *schema2.DeserializedManifest) (distribution.Manifest, error) {
targetDescriptor := schema2Manifest.Target()
blobs := imh.Repository.Blobs(imh)
configJSON, err := blobs.Get(imh, targetDescriptor.Digest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return nil, err
}
ref := imh.Repository.Named()
if imh.Tag != "" {
ref, err = reference.WithTag(ref, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail(err))
return nil, err
}
}
builder := schema1.NewConfigManifestBuilder(imh.Repository.Blobs(imh), imh.Context.App.trustKey, ref, configJSON)
for _, d := range schema2Manifest.References() {
if err := builder.AppendReference(d); err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return nil, err
}
}
manifest, err := builder.Build(imh)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return nil, err
}
imh.Digest = digest.FromBytes(manifest.(*schema1.SignedManifest).Canonical)
return manifest, nil
}
func etagMatch(r *http.Request, etag string) bool {
for _, headerVal := range r.Header["If-None-Match"] {
if headerVal == etag || headerVal == fmt.Sprintf(`"%s"`, etag) { // allow quoted or unquoted
return true
}
}
return false
}
// PutImageManifest validates and stores an image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("PutImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var jsonBuf bytes.Buffer
if err := copyFullPayload(w, r, &jsonBuf, imh, "image manifest PUT", &imh.Errors); err != nil {
// copyFullPayload reports the error if necessary
return
}
mediaType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes())
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return
}
if imh.Digest != "" {
if desc.Digest != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest)
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
}
} else if imh.Tag != "" {
imh.Digest = desc.Digest
} else {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified"))
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
_, err = manifests.Put(imh, manifest, options...)
if err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported)
return
}
if err == distribution.ErrAccessDenied {
imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied)
return
}
switch err := err.(type) {
case distribution.ErrManifestVerification:
for _, verificationError := range err {
switch verificationError := verificationError.(type) {
case distribution.ErrManifestBlobUnknown:
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestBlobUnknown.WithDetail(verificationError.Digest))
case distribution.ErrManifestNameInvalid:
imh.Errors = append(imh.Errors, v2.ErrorCodeNameInvalid.WithDetail(err))
case distribution.ErrManifestUnverified:
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnverified)
default:
if verificationError == digest.ErrDigestInvalidFormat {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown, verificationError)
}
}
}
case errcode.Error:
imh.Errors = append(imh.Errors, err)
default:
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Construct a canonical url for the uploaded manifest.
ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
location, err := imh.urlBuilder.BuildManifestURL(ref)
if err != nil {
// NOTE(stevvooe): Given the behavior above, this absurdly unlikely to
// happen. We'll log the error here but proceed as if it worked. Worst
// case, we set an empty location header.
ctxu.GetLogger(imh).Errorf("error building manifest url from digest: %v", err)
}
w.Header().Set("Location", location)
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.WriteHeader(http.StatusCreated)
}
// DeleteImageManifest removes the manifest with the given digest from the registry.
func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("DeleteImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
err = manifests.Delete(imh, imh.Digest)
if err != nil {
switch err {
case digest.ErrDigestUnsupported:
case digest.ErrDigestInvalidFormat:
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
case distribution.ErrBlobUnknown:
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown)
return
case distribution.ErrUnsupported:
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported)
return
default:
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown)
return
}
}
tagService := imh.Repository.Tags(imh)
referencedTags, err := tagService.Lookup(imh, distribution.Descriptor{Digest: imh.Digest})
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
for _, tag := range referencedTags {
if err := tagService.Untag(imh, tag); err != nil {
imh.Errors = append(imh.Errors, err)
return
}
}
w.WriteHeader(http.StatusAccepted)
}

45
docs/handlers/mail.go
View file

@ -1,45 +0,0 @@
package handlers
import (
"errors"
"net/smtp"
"strings"
)
// mailer provides fields of email configuration for sending.
type mailer struct {
Addr, Username, Password, From string
Insecure bool
To []string
}
// sendMail allows users to send email, only if mail parameters is configured correctly.
func (mail *mailer) sendMail(subject, message string) error {
addr := strings.Split(mail.Addr, ":")
if len(addr) != 2 {
return errors.New("Invalid Mail Address")
}
host := addr[0]
msg := []byte("To:" + strings.Join(mail.To, ";") +
"\r\nFrom: " + mail.From +
"\r\nSubject: " + subject +
"\r\nContent-Type: text/plain\r\n\r\n" +
message)
auth := smtp.PlainAuth(
"",
mail.Username,
mail.Password,
host,
)
err := smtp.SendMail(
mail.Addr,
auth,
mail.From,
mail.To,
[]byte(msg),
)
if err != nil {
return err
}
return nil
}

62
docs/handlers/tags.go
View file

@ -1,62 +0,0 @@
package handlers
import (
"encoding/json"
"net/http"
"github.com/docker/distribution"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
)
// tagsDispatcher constructs the tags handler api endpoint.
func tagsDispatcher(ctx *Context, r *http.Request) http.Handler {
tagsHandler := &tagsHandler{
Context: ctx,
}
return handlers.MethodHandler{
"GET": http.HandlerFunc(tagsHandler.GetTags),
}
}
// tagsHandler handles requests for lists of tags under a repository name.
type tagsHandler struct {
*Context
}
type tagsAPIResponse struct {
Name string `json:"name"`
Tags []string `json:"tags"`
}
// GetTags returns a json list of tags for a specific image name.
func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
tagService := th.Repository.Tags(th)
tags, err := tagService.All(th)
if err != nil {
switch err := err.(type) {
case distribution.ErrRepositoryUnknown:
th.Errors = append(th.Errors, v2.ErrorCodeNameUnknown.WithDetail(map[string]string{"name": th.Repository.Named().Name()}))
case errcode.Error:
th.Errors = append(th.Errors, err)
default:
th.Errors = append(th.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
enc := json.NewEncoder(w)
if err := enc.Encode(tagsAPIResponse{
Name: th.Repository.Named().Name(),
Tags: tags,
}); err != nil {
th.Errors = append(th.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}

24
docs/help.md Normal file
View file

@ -0,0 +1,24 @@
<!--[metadata]>
+++
title = "Getting help"
description = "Getting help with the Registry"
keywords = ["registry, on-prem, images, tags, repository, distribution, help, 101, TL;DR"]
[menu.main]
parent="smn_registry"
weight=9
+++
<![end-metadata]-->
# Getting help
If you need help, or just want to chat, you can reach us:
- on irc: `#docker-distribution` on freenode
- on the [mailing list](https://groups.google.com/a/dockerproject.org/forum/#!forum/distribution) (mail at <distribution@dockerproject.org>)
If you want to report a bug:
- be sure to first read about [how to contribute](https://github.com/docker/distribution/blob/master/CONTRIBUTING.md)
- you can then do so on the [GitHub project bugtracker](https://github.com/docker/distribution/issues)
You can also find out more about the Docker's project [Getting Help resources](/opensource/get-help.md).

1
docs/images/notifications.gliffy Normal file
View file

File diff suppressed because one or more lines are too long

BIN
docs/images/notifications.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 37 KiB

1
docs/images/notifications.svg Normal file
View file

File diff suppressed because one or more lines are too long
Side by side

After

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/v2-registry-auth.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

67
docs/index.md Normal file
View file

@ -0,0 +1,67 @@
<!--[metadata]>
+++
title = "Registry Overview"
description = "High-level overview of the Registry"
keywords = ["registry, on-prem, images, tags, repository, distribution"]
aliases = ["/registry/overview/"]
[menu.main]
parent="smn_registry"
weight=1
+++
<![end-metadata]-->
# Docker Registry
## What it is
The Registry is a stateless, highly scalable server side application that stores and lets you distribute Docker images.
The Registry is open-source, under the permissive [Apache license](http://en.wikipedia.org/wiki/Apache_License).
## Why use it
You should use the Registry if you want to:
* tightly control where your images are being stored
* fully own your images distribution pipeline
* integrate image storage and distribution tightly into your in-house development workflow
## Alternatives
Users looking for a zero maintenance, ready-to-go solution are encouraged to head-over to the [Docker Hub](https://hub.docker.com), which provides a free-to-use, hosted Registry, plus additional features (organization accounts, automated builds, and more).
Users looking for a commercially supported version of the Registry should look into [Docker Trusted Registry](https://docs.docker.com/docker-trusted-registry/overview/).
## Requirements
The Registry is compatible with Docker engine **version 1.6.0 or higher**.
If you really need to work with older Docker versions, you should look into the [old python registry](https://github.com/docker/docker-registry).
## TL;DR
Start your registry
docker run -d -p 5000:5000 --name registry registry:2
Pull (or build) some image from the hub
docker pull ubuntu
Tag the image so that it points to your registry
docker tag ubuntu localhost:5000/myfirstimage
Push it
docker push localhost:5000/myfirstimage
Pull it back
docker pull localhost:5000/myfirstimage
Now stop your registry and remove all data
docker stop registry && docker rm -v registry
## Next
You should now read the [detailed introduction about the registry](introduction.md), or jump directly to [deployment instructions](deploying.md).

114
docs/insecure.md Normal file
View file

@ -0,0 +1,114 @@
<!--[metadata]>
+++
title = "Testing an insecure registry"
description = "Deploying a Registry in an insecure fashion"
keywords = ["registry, on-prem, images, tags, repository, distribution, insecure"]
[menu.main]
parent="smn_registry_ref"
weight=5
+++
<![end-metadata]-->
# Insecure Registry
While it's highly recommended to secure your registry using a TLS certificate
issued by a known CA, you may alternatively decide to use self-signed
certificates, or even use your registry over plain http.
You have to understand the downsides in doing so, and the extra burden in
configuration.
## Deploying a plain HTTP registry
> **Warning**: it's not possible to use an insecure registry with basic authentication.
This basically tells Docker to entirely disregard security for your registry.
While this is relatively easy to configure the daemon in this way, it is
**very** insecure. It does expose your registry to trivial MITM. Only use this
solution for isolated testing or in a tightly controlled, air-gapped
environment.
1. Open the `/etc/default/docker` file or `/etc/sysconfig/docker` for editing.
Depending on your operating system, your Engine daemon start options.
2. Edit (or add) the `DOCKER_OPTS` line and add the `--insecure-registry` flag.
This flag takes the URL of your registry, for example.
`DOCKER_OPTS="--insecure-registry myregistrydomain.com:5000"`
3. Close and save the configuration file.
4. Restart your Docker daemon
The command you use to restart the daemon depends on your operating system.
For example, on Ubuntu, this is usually the `service docker stop` and `service
docker start` command.
5. Repeat this configuration on every Engine host that wants to access your registry.
## Using self-signed certificates
> **Warning**: using this along with basic authentication requires to **also** trust the certificate into the OS cert store for some versions of docker (see below)
This is more secure than the insecure registry solution. You must configure every docker daemon that wants to access your registry
1. Generate your own certificate:
mkdir -p certs && openssl req \
-newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key \
-x509 -days 365 -out certs/domain.crt
2. Be sure to use the name `myregistrydomain.com` as a CN.
3. Use the result to [start your registry with TLS enabled](./deploying.md#get-a-certificate)
4. Instruct every docker daemon to trust that certificate.
This is done by copying the `domain.crt` file to `/etc/docker/certs.d/myregistrydomain.com:5000/ca.crt`.
5. Don't forget to restart the Engine daemon.
## Troubleshooting insecure registry
This sections lists some common failures and how to recover from them.
### Failing...
Failing to configure the Engine daemon and trying to pull from a registry that is not using
TLS will results in the following message:
```
FATA[0000] Error response from daemon: v1 ping attempt failed with error:
Get https://myregistrydomain.com:5000/v1/_ping: tls: oversized record received with length 20527.
If this private registry supports only HTTP or HTTPS with an unknown CA certificate,please add
`--insecure-registry myregistrydomain.com:5000` to the daemon's arguments.
In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag;
simply place the CA certificate at /etc/docker/certs.d/myregistrydomain.com:5000/ca.crt
```
### Docker still complains about the certificate when using authentication?
When using authentication, some versions of docker also require you to trust the certificate at the OS level. Usually, on Ubuntu this is done with:
```bash
$ cp certs/domain.crt /usr/local/share/ca-certificates/myregistrydomain.com.crt
update-ca-certificates
```
... and on Red Hat (and its derivatives) with:
```bash
cp certs/domain.crt /etc/pki/ca-trust/source/anchors/myregistrydomain.com.crt
update-ca-trust
```
... On some distributions, e.g. Oracle Linux 6, the Shared System Certificates feature needs to be manually enabled:
```bash
$ update-ca-trust enable
```
Now restart docker (`service docker stop && service docker start`, or any other way you use to restart docker).

55
docs/introduction.md Normal file
View file

@ -0,0 +1,55 @@
<!--[metadata]>
+++
title = "Understanding the Registry"
description = "Explains what the Registry is, basic use cases and requirements"
keywords = ["registry, on-prem, images, tags, repository, distribution, use cases, requirements"]
[menu.main]
parent="smn_registry"
weight=2
+++
<![end-metadata]-->
# Understanding the Registry
A registry is a storage and content delivery system, holding named Docker images, available in different tagged versions.
> Example: the image `distribution/registry`, with tags `2.0` and `2.1`.
Users interact with a registry by using docker push and pull commands.
> Example: `docker pull registry-1.docker.io/distribution/registry:2.1`.
Storage itself is delegated to drivers. The default storage driver is the local posix filesystem, which is suitable for development or small deployments. Additional cloud-based storage drivers like S3, Microsoft Azure, OpenStack Swift and Aliyun OSS are also supported. People looking into using other storage backends may do so by writing their own driver implementing the [Storage API](storage-drivers/index.md).
Since securing access to your hosted images is paramount, the Registry natively supports TLS and basic authentication.
The Registry GitHub repository includes additional information about advanced authentication and authorization methods. Only very large or public deployments are expected to extend the Registry in this way.
Finally, the Registry ships with a robust [notification system](notifications.md), calling webhooks in response to activity, and both extensive logging and reporting, mostly useful for large installations that want to collect metrics.
## Understanding image naming
Image names as used in typical docker commands reflect their origin:
* `docker pull ubuntu` instructs docker to pull an image named `ubuntu` from the official Docker Hub. This is simply a shortcut for the longer `docker pull docker.io/library/ubuntu` command
* `docker pull myregistrydomain:port/foo/bar` instructs docker to contact the registry located at `myregistrydomain:port` to find the image `foo/bar`
You can find out more about the various Docker commands dealing with images in the [official Docker engine documentation](/engine/reference/commandline/cli.md).
## Use cases
Running your own Registry is a great solution to integrate with and complement your CI/CD system. In a typical workflow, a commit to your source revision control system would trigger a build on your CI system, which would then push a new image to your Registry if the build is successful. A notification from the Registry would then trigger a deployment on a staging environment, or notify other systems that a new image is available.
It's also an essential component if you want to quickly deploy a new image over a large cluster of machines.
Finally, it's the best way to distribute images inside an isolated network.
## Requirements
You absolutely need to be familiar with Docker, specifically with regard to pushing and pulling images. You must understand the difference between the daemon and the cli, and at least grasp basic concepts about networking.
Also, while just starting a registry is fairly easy, operating it in a production environment requires operational skills, just like any other service. You are expected to be familiar with systems availability and scalability, logging and log processing, systems monitoring, and security 101. Strong understanding of http and overall network communications, plus familiarity with golang are certainly useful as well for advanced operations or hacking.
## Next
Dive into [deploying your registry](deploying.md)

74
docs/listener/listener.go
View file

@ -1,74 +0,0 @@
package listener
import (
"fmt"
"net"
"os"
"time"
)
// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe and ListenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
// it is a plain copy-paste from net/http/server.go
type tcpKeepAliveListener struct {
*net.TCPListener
}
func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
tc, err := ln.AcceptTCP()
if err != nil {
return
}
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(3 * time.Minute)
return tc, nil
}
// NewListener announces on laddr and net. Accepted values of the net are
// 'unix' and 'tcp'
func NewListener(net, laddr string) (net.Listener, error) {
switch net {
case "unix":
return newUnixListener(laddr)
case "tcp", "": // an empty net means tcp
return newTCPListener(laddr)
default:
return nil, fmt.Errorf("unknown address type %s", net)
}
}
func newUnixListener(laddr string) (net.Listener, error) {
fi, err := os.Stat(laddr)
if err == nil {
// the file exists.
// try to remove it if it's a socket
if !isSocket(fi.Mode()) {
return nil, fmt.Errorf("file %s exists and is not a socket", laddr)
}
if err := os.Remove(laddr); err != nil {
return nil, err
}
} else if !os.IsNotExist(err) {
// we can't do stat on the file.
// it means we can not remove it
return nil, err
}
return net.Listen("unix", laddr)
}
func isSocket(m os.FileMode) bool {
return m&os.ModeSocket != 0
}
func newTCPListener(laddr string) (net.Listener, error) {
ln, err := net.Listen("tcp", laddr)
if err != nil {
return nil, err
}
return tcpKeepAliveListener{ln.(*net.TCPListener)}, nil
}

23
docs/menu.md Normal file
View file

@ -0,0 +1,23 @@
<!--[metadata]>
+++
title = "Docker Registry"
description = "High-level overview of the Registry"
keywords = ["registry, on-prem, images, tags, repository, distribution"]
type = "menu"
[menu.main]
identifier="smn_registry"
parent="mn_components"
+++
<![end-metadata]-->
# Overview of Docker Registry Documentation
The Docker Registry documentation includes the following topics:
* [Docker Registry Introduction](index.md)
* [Understanding the Registry](introduction.md)
* [Deploying a registry server](deploying.md)
* [Registry Configuration Reference](configuration.md)
* [Notifications](notifications.md)
* [Recipes](recipes/index.md)
* [Getting help](help.md)

54
docs/middleware/registry/middleware.go
View file

@ -1,54 +0,0 @@
package middleware
import (
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage"
)
// InitFunc is the type of a RegistryMiddleware factory function and is
// used to register the constructor for different RegistryMiddleware backends.
type InitFunc func(ctx context.Context, registry distribution.Namespace, options map[string]interface{}) (distribution.Namespace, error)
var middlewares map[string]InitFunc
var registryoptions []storage.RegistryOption
// Register is used to register an InitFunc for
// a RegistryMiddleware backend with the given name.
func Register(name string, initFunc InitFunc) error {
if middlewares == nil {
middlewares = make(map[string]InitFunc)
}
if _, exists := middlewares[name]; exists {
return fmt.Errorf("name already registered: %s", name)
}
middlewares[name] = initFunc
return nil
}
// Get constructs a RegistryMiddleware with the given options using the named backend.
func Get(ctx context.Context, name string, options map[string]interface{}, registry distribution.Namespace) (distribution.Namespace, error) {
if middlewares != nil {
if initFunc, exists := middlewares[name]; exists {
return initFunc(ctx, registry, options)
}
}
return nil, fmt.Errorf("no registry middleware registered with name: %s", name)
}
// RegisterOptions adds more options to RegistryOption list. Options get applied before
// any other configuration-based options.
func RegisterOptions(options ...storage.RegistryOption) error {
registryoptions = append(registryoptions, options...)
return nil
}
// GetRegistryOptions returns list of RegistryOption.
func GetRegistryOptions() []storage.RegistryOption {
return registryoptions
}

40
docs/middleware/repository/middleware.go
View file

@ -1,40 +0,0 @@
package middleware
import (
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
// InitFunc is the type of a RepositoryMiddleware factory function and is
// used to register the constructor for different RepositoryMiddleware backends.
type InitFunc func(ctx context.Context, repository distribution.Repository, options map[string]interface{}) (distribution.Repository, error)
var middlewares map[string]InitFunc
// Register is used to register an InitFunc for
// a RepositoryMiddleware backend with the given name.
func Register(name string, initFunc InitFunc) error {
if middlewares == nil {
middlewares = make(map[string]InitFunc)
}
if _, exists := middlewares[name]; exists {
return fmt.Errorf("name already registered: %s", name)
}
middlewares[name] = initFunc
return nil
}
// Get constructs a RepositoryMiddleware with the given options using the named backend.
func Get(ctx context.Context, name string, options map[string]interface{}, repository distribution.Repository) (distribution.Repository, error) {
if middlewares != nil {
if initFunc, exists := middlewares[name]; exists {
return initFunc(ctx, repository, options)
}
}
return nil, fmt.Errorf("no repository middleware registered with name: %s", name)
}

30
docs/migration.md Normal file
View file

@ -0,0 +1,30 @@
<!--[metadata]>
+++
draft = true
+++
<![end-metadata]-->
# Migrating a 1.0 registry to 2.0
TODO: This needs to be revised in light of Olivier's work
A few thoughts here:
There was no "1.0". There was an implementation of the Registry API V1 but only a version 0.9 of the service was released.
The image formats are not compatible in any way. One must convert v1 images to v2 images using a docker client or other tool.
One can migrate images from one version to the other by pulling images from the old registry and pushing them to the v2 registry.
-----
The Docker Registry 2.0 is backward compatible with images created by the earlier specification. If you are migrating a private registry to version 2.0, you should use the following process:
1. Configure and test a 2.0 registry image in a sandbox environment.
2. Back up up your production image storage.
Your production image storage should reside on a volume or storage backend.
Make sure you have a backup of its contents.
3. Stop your existing registry service.
4. Restart your registry with your tested 2.0 image.

350
docs/notifications.md Normal file
View file

@ -0,0 +1,350 @@
<!--[metadata]>
+++
title = "Working with notifications"
description = "Explains how to work with registry notifications"
keywords = ["registry, on-prem, images, tags, repository, distribution, notifications, advanced"]
[menu.main]
parent="smn_registry"
weight=5
+++
<![end-metadata]-->
# Notifications
The Registry supports sending webhook notifications in response to events
happening within the registry. Notifications are sent in response to manifest
pushes and pulls and layer pushes and pulls. These actions are serialized into
events. The events are queued into a registry-internal broadcast system which
queues and dispatches events to [_Endpoints_](#endpoints).
![](images/notifications.png)
## Endpoints
Notifications are sent to _endpoints_ via HTTP requests. Each configured
endpoint has isolated queues, retry configuration and http targets within each
instance of a registry. When an action happens within the registry, it is
converted into an event which is dropped into an inmemory queue. When the
event reaches the end of the queue, an http request is made to the endpoint
until the request succeeds. The events are sent serially to each endpoint but
order is not guaranteed.
## Configuration
To setup a registry instance to send notifications to endpoints, one must add
them to the configuration. A simple example follows:
notifications:
endpoints:
- name: alistener
url: https://mylistener.example.com/event
headers:
Authorization: [Bearer <your token, if needed>]
timeout: 500ms
threshold: 5
backoff: 1s
The above would configure the registry with an endpoint to send events to
`https://mylistener.example.com/event`, with the header "Authorization: Bearer
<your token, if needed>". The request would timeout after 500 milliseconds. If
5 failures happen consecutively, the registry will backoff for 1 second before
trying again.
For details on the fields, please see the [configuration documentation](configuration.md#notifications).
A properly configured endpoint should lead to a log message from the registry
upon startup:
```
INFO[0000] configuring endpoint alistener (https://mylistener.example.com/event), timeout=500ms, headers=map[Authorization:[Bearer <your token if needed>]] app.id=812bfeb2-62d6-43cf-b0c6-152f541618a3 environment=development service=registry
```
## Events
Events have a well-defined JSON structure and are sent as the body of
notification requests. One or more events are sent in a structure called an
envelope. Each event has a unique id that can be used to uniquely identify incoming
requests, if required. Along with that, an _action_ is provided with a
_target_, identifying the object mutated during the event.
The fields available in an `event` are described below.
Field | Type | Description
----- | ----- | -------------
id | string |ID provides a unique identifier for the event.
timestamp | Time | Timestamp is the time at which the event occurred.
action | string | Action indicates what action encompasses the provided event.
target | distribution.Descriptor | Target uniquely describes the target of the event.
length | int | Length in bytes of content. Same as Size field in Descriptor.
repository | string | Repository identifies the named repository.
fromRepository | string | FromRepository identifies the named repository which a blob was mounted from if appropriate.
url | string | URL provides a direct link to the content.
tag | string | Tag identifies a tag name in tag events
request | [RequestRecord](https://godoc.org/github.com/docker/distribution/notifications#RequestRecord) | Request covers the request that generated the event.
actor | [ActorRecord](https://godoc.org/github.com/docker/distribution/notifications#ActorRecord). | Actor specifies the agent that initiated the event. For most situations, this could be from the authorization context of the request.
source | [SourceRecord](https://godoc.org/github.com/docker/distribution/notifications#SourceRecord) | Source identifies the registry node that generated the event. Put differently, while the actor "initiates" the event, the source "generates" it.
The following is an example of a JSON event, sent in response to the push of a
manifest:
```json
{
"events": [
{
"id": "320678d8-ca14-430f-8bb6-4ca139cd83f7",
"timestamp": "2016-03-09T14:44:26.402973972-08:00",
"action": "pull",
"target": {
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"size": 708,
"digest": "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
"length": 708,
"repository": "hello-world",
"url": "http://192.168.100.227:5000/v2/hello-world/manifests/sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
"tag": "latest"
},
"request": {
"id": "6df24a34-0959-4923-81ca-14f09767db19",
"addr": "192.168.64.11:42961",
"host": "192.168.100.227:5000",
"method": "GET",
"useragent": "curl/7.38.0"
},
"actor": {},
"source": {
"addr": "xtal.local:5000",
"instanceID": "a53db899-3b4b-4a62-a067-8dd013beaca4"
}
}
]
}
```
The target struct of events which are sent when manifests and blobs are deleted
will contain a subset of the data contained in Get and Put events. Specifically,
only the digest and repository will be sent.
```json
"target": {
"digest": "sha256:d89e1bee20d9cb344674e213b581f14fbd8e70274ecf9d10c514bab78a307845",
"repository": "library/test"
},
```
> __NOTE:__ As of version 2.1, the `length` field for event targets
> is being deprecated for the `size` field, bringing the target in line with
> common nomenclature. Both will continue to be set for the foreseeable
> future. Newer code should favor `size` but accept either.
## Envelope
The envelope contains one or more events, with the following json structure:
```json
{
"events": [ ... ],
}
```
While events may be sent in the same envelope, the set of events within that
envelope have no implied relationship. For example, the registry may choose to
group unrelated events and send them in the same envelope to reduce the total
number of requests.
The full package has the mediatype
"application/vnd.docker.distribution.events.v1+json", which will be set on the
request coming to an endpoint.
An example of a full event may look as follows:
```json
GET /callback
Host: application/vnd.docker.distribution.events.v1+json
Authorization: Bearer <your token, if needed>
Content-Type: application/vnd.docker.distribution.events.v1+json
{
"events": [
{
"id": "asdf-asdf-asdf-asdf-0",
"timestamp": "2006-01-02T15:04:05Z",
"action": "push",
"target": {
"mediaType": "application/vnd.docker.distribution.manifest.v1+json",
"length": 1,
"digest": "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
"repository": "library/test",
"url": "http://example.com/v2/library/test/manifests/sha256:c3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5"
},
"request": {
"id": "asdfasdf",
"addr": "client.local",
"host": "registrycluster.local",
"method": "PUT",
"useragent": "test/0.1"
},
"actor": {
"name": "test-actor"
},
"source": {
"addr": "hostname.local:port"
}
},
{
"id": "asdf-asdf-asdf-asdf-1",
"timestamp": "2006-01-02T15:04:05Z",
"action": "push",
"target": {
"mediaType": "application/vnd.docker.container.image.rootfs.diff+x-gtar",
"length": 2,
"digest": "sha256:c3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5",
"repository": "library/test",
"url": "http://example.com/v2/library/test/blobs/sha256:c3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5"
},
"request": {
"id": "asdfasdf",
"addr": "client.local",
"host": "registrycluster.local",
"method": "PUT",
"useragent": "test/0.1"
},
"actor": {
"name": "test-actor"
},
"source": {
"addr": "hostname.local:port"
}
},
{
"id": "asdf-asdf-asdf-asdf-2",
"timestamp": "2006-01-02T15:04:05Z",
"action": "push",
"target": {
"mediaType": "application/vnd.docker.container.image.rootfs.diff+x-gtar",
"length": 3,
"digest": "sha256:c3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5",
"repository": "library/test",
"url": "http://example.com/v2/library/test/blobs/sha256:c3b3692957d439ac1928219a83fac91e7bf96c153725526874673ae1f2023f8d5"
},
"request": {
"id": "asdfasdf",
"addr": "client.local",
"host": "registrycluster.local",
"method": "PUT",
"useragent": "test/0.1"
},
"actor": {
"name": "test-actor"
},
"source": {
"addr": "hostname.local:port"
}
}
]
}
```
## Responses
The registry is fairly accepting of the response codes from endpoints. If an
endpoint responds with any 2xx or 3xx response code (after following
redirects), the message will be considered delivered and discarded.
In turn, it is recommended that endpoints are accepting of incoming responses,
as well. While the format of event envelopes are standardized by media type,
any "pickyness" about validation may cause the queue to backup on the
registry.
## Monitoring
The state of the endpoints are reported via the debug/vars http interface,
usually configured to `http://localhost:5001/debug/vars`. Information such as
configuration and metrics are available by endpoint.
The following provides an example of a few endpoints that have experienced
several failures and have since recovered:
```json
"notifications":{
"endpoints":[
{
"name":"local-5003",
"url":"http://localhost:5003/callback",
"Headers":{
"Authorization":[
"Bearer \u003can example token\u003e"
]
},
"Timeout":1000000000,
"Threshold":10,
"Backoff":1000000000,
"Metrics":{
"Pending":76,
"Events":76,
"Successes":0,
"Failures":0,
"Errors":46,
"Statuses":{
}
}
},
{
"name":"local-8083",
"url":"http://localhost:8083/callback",
"Headers":null,
"Timeout":1000000000,
"Threshold":10,
"Backoff":1000000000,
"Metrics":{
"Pending":0,
"Events":76,
"Successes":76,
"Failures":0,
"Errors":28,
"Statuses":{
"202 Accepted":76
}
}
}
]
}
```
If using notification as part of a larger application, it is _critical_ to
monitor the size ("Pending" above) of the endpoint queues. If failures or
queue sizes are increasing, it can indicate a larger problem.
The logs are also a valuable resource for monitoring problems. A failing
endpoint will lead to messages similar to the following:
```
ERRO[0340] retryingsink: error writing events: httpSink{http://localhost:5003/callback}: error posting: Post http://localhost:5003/callback: dial tcp 127.0.0.1:5003: connection refused, retrying
WARN[0340] httpSink{http://localhost:5003/callback} encountered too many errors, backing off
```
The above indicates that several errors have led to a backoff and the registry
will wait before retrying.
## Considerations
Currently, the queues are inmemory, so endpoints should be _reasonably
reliable_. They are designed to make a best-effort to send the messages but if
an instance is lost, messages may be dropped. If an endpoint goes down, care
should be taken to ensure that the registry instance is not terminated before
the endpoint comes back up or messages will be lost.
This can be mitigated by running endpoints in close proximity to the registry
instances. One could run an endpoint that pages to disk and then forwards a
request to provide better durability.
The notification system is designed around a series of interchangeable _sinks_
which can be wired up to achieve interesting behavior. If this system doesn't
provide acceptable guarantees, adding a transactional `Sink` to the registry
is a possibility, although it may have an effect on request service time.
Please see the
[godoc](http://godoc.org/github.com/docker/distribution/notifications#Sink)
for more information.

58
docs/proxy/proxyauth.go
View file

@ -1,58 +0,0 @@
package proxy
import (
"net/http"
"net/url"
"github.com/docker/distribution/registry/client/auth"
)
const tokenURL = "https://auth.docker.io/token"
const challengeHeader = "Docker-Distribution-Api-Version"
type userpass struct {
username string
password string
}
type credentials struct {
creds map[string]userpass
}
func (c credentials) Basic(u *url.URL) (string, string) {
up := c.creds[u.String()]
return up.username, up.password
}
func (c credentials) RefreshToken(u *url.URL, service string) string {
return ""
}
func (c credentials) SetRefreshToken(u *url.URL, service, token string) {
}
// configureAuth stores credentials for challenge responses
func configureAuth(username, password string) (auth.CredentialStore, error) {
creds := map[string]userpass{
tokenURL: {
username: username,
password: password,
},
}
return credentials{creds: creds}, nil
}
func ping(manager auth.ChallengeManager, endpoint, versionHeader string) error {
resp, err := http.Get(endpoint)
if err != nil {
return err
}
defer resp.Body.Close()
if err := manager.AddResponse(resp); err != nil {
return err
}
return nil
}

222
docs/proxy/proxyblobstore.go
View file

@ -1,222 +0,0 @@
package proxy
import (
"io"
"net/http"
"strconv"
"sync"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/proxy/scheduler"
)
// todo(richardscothern): from cache control header or config file
const blobTTL = time.Duration(24 * 7 * time.Hour)
type proxyBlobStore struct {
localStore distribution.BlobStore
remoteStore distribution.BlobService
scheduler *scheduler.TTLExpirationScheduler
repositoryName reference.Named
authChallenger authChallenger
}
var _ distribution.BlobStore = &proxyBlobStore{}
// inflight tracks currently downloading blobs
var inflight = make(map[digest.Digest]struct{})
// mu protects inflight
var mu sync.Mutex
func setResponseHeaders(w http.ResponseWriter, length int64, mediaType string, digest digest.Digest) {
w.Header().Set("Content-Length", strconv.FormatInt(length, 10))
w.Header().Set("Content-Type", mediaType)
w.Header().Set("Docker-Content-Digest", digest.String())
w.Header().Set("Etag", digest.String())
}
func (pbs *proxyBlobStore) copyContent(ctx context.Context, dgst digest.Digest, writer io.Writer) (distribution.Descriptor, error) {
desc, err := pbs.remoteStore.Stat(ctx, dgst)
if err != nil {
return distribution.Descriptor{}, err
}
if w, ok := writer.(http.ResponseWriter); ok {
setResponseHeaders(w, desc.Size, desc.MediaType, dgst)
}
remoteReader, err := pbs.remoteStore.Open(ctx, dgst)
if err != nil {
return distribution.Descriptor{}, err
}
_, err = io.CopyN(writer, remoteReader, desc.Size)
if err != nil {
return distribution.Descriptor{}, err
}
proxyMetrics.BlobPush(uint64(desc.Size))
return desc, nil
}
func (pbs *proxyBlobStore) serveLocal(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) (bool, error) {
localDesc, err := pbs.localStore.Stat(ctx, dgst)
if err != nil {
// Stat can report a zero sized file here if it's checked between creation
// and population. Return nil error, and continue
return false, nil
}
if err == nil {
proxyMetrics.BlobPush(uint64(localDesc.Size))
return true, pbs.localStore.ServeBlob(ctx, w, r, dgst)
}
return false, nil
}
func (pbs *proxyBlobStore) storeLocal(ctx context.Context, dgst digest.Digest) error {
defer func() {
mu.Lock()
delete(inflight, dgst)
mu.Unlock()
}()
var desc distribution.Descriptor
var err error
var bw distribution.BlobWriter
bw, err = pbs.localStore.Create(ctx)
if err != nil {
return err
}
desc, err = pbs.copyContent(ctx, dgst, bw)
if err != nil {
return err
}
_, err = bw.Commit(ctx, desc)
if err != nil {
return err
}
return nil
}
func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
served, err := pbs.serveLocal(ctx, w, r, dgst)
if err != nil {
context.GetLogger(ctx).Errorf("Error serving blob from local storage: %s", err.Error())
return err
}
if served {
return nil
}
if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
return err
}
mu.Lock()
_, ok := inflight[dgst]
if ok {
mu.Unlock()
_, err := pbs.copyContent(ctx, dgst, w)
return err
}
inflight[dgst] = struct{}{}
mu.Unlock()
go func(dgst digest.Digest) {
if err := pbs.storeLocal(ctx, dgst); err != nil {
context.GetLogger(ctx).Errorf("Error committing to storage: %s", err.Error())
}
blobRef, err := reference.WithDigest(pbs.repositoryName, dgst)
if err != nil {
context.GetLogger(ctx).Errorf("Error creating reference: %s", err)
return
}
pbs.scheduler.AddBlob(blobRef, repositoryTTL)
}(dgst)
_, err = pbs.copyContent(ctx, dgst, w)
if err != nil {
return err
}
return nil
}
func (pbs *proxyBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
desc, err := pbs.localStore.Stat(ctx, dgst)
if err == nil {
return desc, err
}
if err != distribution.ErrBlobUnknown {
return distribution.Descriptor{}, err
}
if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
return distribution.Descriptor{}, err
}
return pbs.remoteStore.Stat(ctx, dgst)
}
func (pbs *proxyBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
blob, err := pbs.localStore.Get(ctx, dgst)
if err == nil {
return blob, nil
}
if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
return []byte{}, err
}
blob, err = pbs.remoteStore.Get(ctx, dgst)
if err != nil {
return []byte{}, err
}
_, err = pbs.localStore.Put(ctx, "", blob)
if err != nil {
return []byte{}, err
}
return blob, nil
}
// Unsupported functions
func (pbs *proxyBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
return distribution.Descriptor{}, distribution.ErrUnsupported
}
func (pbs *proxyBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
return nil, distribution.ErrUnsupported
}
func (pbs *proxyBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
return nil, distribution.ErrUnsupported
}
func (pbs *proxyBlobStore) Mount(ctx context.Context, sourceRepo reference.Named, dgst digest.Digest) (distribution.Descriptor, error) {
return distribution.Descriptor{}, distribution.ErrUnsupported
}
func (pbs *proxyBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
return nil, distribution.ErrUnsupported
}
func (pbs *proxyBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
return distribution.ErrUnsupported
}

409
docs/proxy/proxyblobstore_test.go
View file

@ -1,409 +0,0 @@
package proxy
import (
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/proxy/scheduler"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/cache/memory"
"github.com/docker/distribution/registry/storage/driver/filesystem"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
var sbsMu sync.Mutex
type statsBlobStore struct {
stats map[string]int
blobs distribution.BlobStore
}
func (sbs statsBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
sbsMu.Lock()
sbs.stats["put"]++
sbsMu.Unlock()
return sbs.blobs.Put(ctx, mediaType, p)
}
func (sbs statsBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
sbsMu.Lock()
sbs.stats["get"]++
sbsMu.Unlock()
return sbs.blobs.Get(ctx, dgst)
}
func (sbs statsBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
sbsMu.Lock()
sbs.stats["create"]++
sbsMu.Unlock()
return sbs.blobs.Create(ctx, options...)
}
func (sbs statsBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
sbsMu.Lock()
sbs.stats["resume"]++
sbsMu.Unlock()
return sbs.blobs.Resume(ctx, id)
}
func (sbs statsBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
sbsMu.Lock()
sbs.stats["open"]++
sbsMu.Unlock()
return sbs.blobs.Open(ctx, dgst)
}
func (sbs statsBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
sbsMu.Lock()
sbs.stats["serveblob"]++
sbsMu.Unlock()
return sbs.blobs.ServeBlob(ctx, w, r, dgst)
}
func (sbs statsBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
sbsMu.Lock()
sbs.stats["stat"]++
sbsMu.Unlock()
return sbs.blobs.Stat(ctx, dgst)
}
func (sbs statsBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
sbsMu.Lock()
sbs.stats["delete"]++
sbsMu.Unlock()
return sbs.blobs.Delete(ctx, dgst)
}
type testEnv struct {
numUnique int
inRemote []distribution.Descriptor
store proxyBlobStore
ctx context.Context
}
func (te *testEnv) LocalStats() *map[string]int {
sbsMu.Lock()
ls := te.store.localStore.(statsBlobStore).stats
sbsMu.Unlock()
return &ls
}
func (te *testEnv) RemoteStats() *map[string]int {
sbsMu.Lock()
rs := te.store.remoteStore.(statsBlobStore).stats
sbsMu.Unlock()
return &rs
}
// Populate remote store and record the digests
func makeTestEnv(t *testing.T, name string) *testEnv {
nameRef, err := reference.ParseNamed(name)
if err != nil {
t.Fatalf("unable to parse reference: %s", err)
}
ctx := context.Background()
truthDir, err := ioutil.TempDir("", "truth")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
cacheDir, err := ioutil.TempDir("", "cache")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
localDriver, err := filesystem.FromParameters(map[string]interface{}{
"rootdirectory": truthDir,
})
if err != nil {
t.Fatalf("unable to create filesystem driver: %s", err)
}
// todo: create a tempfile area here
localRegistry, err := storage.NewRegistry(ctx, localDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
cacheDriver, err := filesystem.FromParameters(map[string]interface{}{
"rootdirectory": cacheDir,
})
if err != nil {
t.Fatalf("unable to create filesystem driver: %s", err)
}
truthRegistry, err := storage.NewRegistry(ctx, cacheDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: truthRepo.Blobs(ctx),
}
localBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: localRepo.Blobs(ctx),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
proxyBlobStore := proxyBlobStore{
repositoryName: nameRef,
remoteStore: truthBlobs,
localStore: localBlobs,
scheduler: s,
authChallenger: &mockChallenger{},
}
te := &testEnv{
store: proxyBlobStore,
ctx: ctx,
}
return te
}
func makeBlob(size int) []byte {
blob := make([]byte, size, size)
for i := 0; i < size; i++ {
blob[i] = byte('A' + rand.Int()%48)
}
return blob
}
func init() {
rand.Seed(42)
}
func perm(m []distribution.Descriptor) []distribution.Descriptor {
for i := 0; i < len(m); i++ {
j := rand.Intn(i + 1)
tmp := m[i]
m[i] = m[j]
m[j] = tmp
}
return m
}
func populate(t *testing.T, te *testEnv, blobCount, size, numUnique int) {
var inRemote []distribution.Descriptor
for i := 0; i < numUnique; i++ {
bytes := makeBlob(size)
for j := 0; j < blobCount/numUnique; j++ {
desc, err := te.store.remoteStore.Put(te.ctx, "", bytes)
if err != nil {
t.Fatalf("Put in store")
}
inRemote = append(inRemote, desc)
}
}
te.inRemote = inRemote
te.numUnique = numUnique
}
func TestProxyStoreGet(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
localStats := te.LocalStats()
remoteStats := te.RemoteStats()
populate(t, te, 1, 10, 1)
_, err := te.store.Get(te.ctx, te.inRemote[0].Digest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 1 && (*localStats)["put"] != 1 {
t.Errorf("Unexpected local counts")
}
if (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected remote get count")
}
_, err = te.store.Get(te.ctx, te.inRemote[0].Digest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 2 && (*localStats)["put"] != 1 {
t.Errorf("Unexpected local counts")
}
if (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected remote get count")
}
}
func TestProxyStoreStat(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
remoteBlobCount := 1
populate(t, te, remoteBlobCount, 10, 1)
localStats := te.LocalStats()
remoteStats := te.RemoteStats()
// Stat - touches both stores
for _, d := range te.inRemote {
_, err := te.store.Stat(te.ctx, d.Digest)
if err != nil {
t.Fatalf("Error stating proxy store")
}
}
if (*localStats)["stat"] != remoteBlobCount {
t.Errorf("Unexpected local stat count")
}
if (*remoteStats)["stat"] != remoteBlobCount {
t.Errorf("Unexpected remote stat count")
}
if te.store.authChallenger.(*mockChallenger).count != len(te.inRemote) {
t.Fatalf("Unexpected auth challenge count, got %#v", te.store.authChallenger)
}
}
func TestProxyStoreServeHighConcurrency(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
blobSize := 200
blobCount := 10
numUnique := 1
populate(t, te, blobCount, blobSize, numUnique)
numClients := 16
testProxyStoreServe(t, te, numClients)
}
func TestProxyStoreServeMany(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
blobSize := 200
blobCount := 10
numUnique := 4
populate(t, te, blobCount, blobSize, numUnique)
numClients := 4
testProxyStoreServe(t, te, numClients)
}
// todo(richardscothern): blobCount must be smaller than num clients
func TestProxyStoreServeBig(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
blobSize := 2 << 20
blobCount := 4
numUnique := 2
populate(t, te, blobCount, blobSize, numUnique)
numClients := 4
testProxyStoreServe(t, te, numClients)
}
// testProxyStoreServe will create clients to consume all blobs
// populated in the truth store
func testProxyStoreServe(t *testing.T, te *testEnv, numClients int) {
localStats := te.LocalStats()
remoteStats := te.RemoteStats()
var wg sync.WaitGroup
for i := 0; i < numClients; i++ {
// Serveblob - pulls through blobs
wg.Add(1)
go func() {
defer wg.Done()
for _, remoteBlob := range te.inRemote {
w := httptest.NewRecorder()
r, err := http.NewRequest("GET", "", nil)
if err != nil {
t.Fatal(err)
}
err = te.store.ServeBlob(te.ctx, w, r, remoteBlob.Digest)
if err != nil {
t.Fatalf(err.Error())
}
bodyBytes := w.Body.Bytes()
localDigest := digest.FromBytes(bodyBytes)
if localDigest != remoteBlob.Digest {
t.Fatalf("Mismatching blob fetch from proxy")
}
}
}()
}
wg.Wait()
remoteBlobCount := len(te.inRemote)
if (*localStats)["stat"] != remoteBlobCount*numClients && (*localStats)["create"] != te.numUnique {
t.Fatal("Expected: stat:", remoteBlobCount*numClients, "create:", remoteBlobCount)
}
// Wait for any async storage goroutines to finish
time.Sleep(3 * time.Second)
remoteStatCount := (*remoteStats)["stat"]
remoteOpenCount := (*remoteStats)["open"]
// Serveblob - blobs come from local
for _, dr := range te.inRemote {
w := httptest.NewRecorder()
r, err := http.NewRequest("GET", "", nil)
if err != nil {
t.Fatal(err)
}
err = te.store.ServeBlob(te.ctx, w, r, dr.Digest)
if err != nil {
t.Fatalf(err.Error())
}
dl := digest.FromBytes(w.Body.Bytes())
if dl != dr.Digest {
t.Errorf("Mismatching blob fetch from proxy")
}
}
localStats = te.LocalStats()
remoteStats = te.RemoteStats()
// Ensure remote unchanged
if (*remoteStats)["stat"] != remoteStatCount && (*remoteStats)["open"] != remoteOpenCount {
t.Fatalf("unexpected remote stats: %#v", remoteStats)
}
}

95
docs/proxy/proxymanifeststore.go
View file

@ -1,95 +0,0 @@
package proxy
import (
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/proxy/scheduler"
)
// todo(richardscothern): from cache control header or config
const repositoryTTL = time.Duration(24 * 7 * time.Hour)
type proxyManifestStore struct {
ctx context.Context
localManifests distribution.ManifestService
remoteManifests distribution.ManifestService
repositoryName reference.Named
scheduler *scheduler.TTLExpirationScheduler
authChallenger authChallenger
}
var _ distribution.ManifestService = &proxyManifestStore{}
func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(ctx, dgst)
if err != nil {
return false, err
}
if exists {
return true, nil
}
if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil {
return false, err
}
return pms.remoteManifests.Exists(ctx, dgst)
}
func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
// At this point `dgst` was either specified explicitly, or returned by the
// tagstore with the most recent association.
var fromRemote bool
manifest, err := pms.localManifests.Get(ctx, dgst, options...)
if err != nil {
if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil {
return nil, err
}
manifest, err = pms.remoteManifests.Get(ctx, dgst, options...)
if err != nil {
return nil, err
}
fromRemote = true
}
_, payload, err := manifest.Payload()
if err != nil {
return nil, err
}
proxyMetrics.ManifestPush(uint64(len(payload)))
if fromRemote {
proxyMetrics.ManifestPull(uint64(len(payload)))
_, err = pms.localManifests.Put(ctx, manifest)
if err != nil {
return nil, err
}
// Schedule the manifest blob for removal
repoBlob, err := reference.WithDigest(pms.repositoryName, dgst)
if err != nil {
context.GetLogger(ctx).Errorf("Error creating reference: %s", err)
return nil, err
}
pms.scheduler.AddManifest(repoBlob, repositoryTTL)
// Ensure the manifest blob is cleaned up
//pms.scheduler.AddBlob(blobRef, repositoryTTL)
}
return manifest, err
}
func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var d digest.Digest
return d, distribution.ErrUnsupported
}
func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
return distribution.ErrUnsupported
}

274
docs/proxy/proxymanifeststore_test.go
View file

@ -1,274 +0,0 @@
package proxy
import (
"io"
"sync"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/proxy/scheduler"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/cache/memory"
"github.com/docker/distribution/registry/storage/driver/inmemory"
"github.com/docker/distribution/testutil"
"github.com/docker/libtrust"
)
type statsManifest struct {
manifests distribution.ManifestService
stats map[string]int
}
type manifestStoreTestEnv struct {
manifestDigest digest.Digest // digest of the signed manifest in the local storage
manifests proxyManifestStore
}
func (te manifestStoreTestEnv) LocalStats() *map[string]int {
ls := te.manifests.localManifests.(statsManifest).stats
return &ls
}
func (te manifestStoreTestEnv) RemoteStats() *map[string]int {
rs := te.manifests.remoteManifests.(statsManifest).stats
return &rs
}
func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error {
sm.stats["delete"]++
return sm.manifests.Delete(ctx, dgst)
}
func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
sm.stats["exists"]++
return sm.manifests.Exists(ctx, dgst)
}
func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
sm.stats["get"]++
return sm.manifests.Get(ctx, dgst)
}
func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
sm.stats["put"]++
return sm.manifests.Put(ctx, manifest)
}
type mockChallenger struct {
sync.Mutex
count int
}
// Called for remote operations only
func (m *mockChallenger) tryEstablishChallenges(context.Context) error {
m.Lock()
defer m.Unlock()
m.count++
return nil
}
func (m *mockChallenger) credentialStore() auth.CredentialStore {
return nil
}
func (m *mockChallenger) challengeManager() auth.ChallengeManager {
return nil
}
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
nameRef, err := reference.ParseNamed(name)
if err != nil {
t.Fatalf("unable to parse reference: %s", err)
}
k, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
truthRegistry, err := storage.NewRegistry(ctx, inmemory.New(),
storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()),
storage.Schema1SigningKey(k))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
tr, err := truthRepo.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
truthManifests := statsManifest{
manifests: tr,
stats: make(map[string]int),
}
manifestDigest, err := populateRepo(t, ctx, truthRepo, name, tag)
if err != nil {
t.Fatalf(err.Error())
}
localRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption, storage.Schema1SigningKey(k))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
lr, err := localRepo.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
localManifests := statsManifest{
manifests: lr,
stats: make(map[string]int),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
return &manifestStoreTestEnv{
manifestDigest: manifestDigest,
manifests: proxyManifestStore{
ctx: ctx,
localManifests: localManifests,
remoteManifests: truthManifests,
scheduler: s,
repositoryName: nameRef,
authChallenger: &mockChallenger{},
},
}
}
func populateRepo(t *testing.T, ctx context.Context, repository distribution.Repository, name, tag string) (digest.Digest, error) {
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: name,
Tag: tag,
}
for i := 0; i < 2; i++ {
wr, err := repository.Blobs(ctx).Create(ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
rs, ts, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ts)
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, err := schema1.Sign(&m, pk)
if err != nil {
t.Fatalf("error signing manifest: %v", err)
}
ms, err := repository.Manifests(ctx)
if err != nil {
t.Fatalf(err.Error())
}
dgst, err := ms.Put(ctx, sm)
if err != nil {
t.Fatalf("unexpected errors putting manifest: %v", err)
}
return dgst, nil
}
// TestProxyManifests contains basic acceptance tests
// for the pull-through behavior
func TestProxyManifests(t *testing.T) {
name := "foo/bar"
env := newManifestStoreTestEnv(t, name, "latest")
localStats := env.LocalStats()
remoteStats := env.RemoteStats()
ctx := context.Background()
// Stat - must check local and remote
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existence")
}
if !exists {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats)
}
if env.manifests.authChallenger.(*mockChallenger).count != 1 {
t.Fatalf("Expected 1 auth challenge, got %#v", env.manifests.authChallenger)
}
// Get - should succeed and pull manifest into local
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected get count")
}
if (*localStats)["put"] != 1 {
t.Errorf("Expected local put")
}
if env.manifests.authChallenger.(*mockChallenger).count != 2 {
t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
}
// Stat - should only go to local
exists, err = env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count")
}
if env.manifests.authChallenger.(*mockChallenger).count != 2 {
t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
}
// Get proxied - won't require another authchallenge
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if env.manifests.authChallenger.(*mockChallenger).count != 2 {
t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
}
}

74
docs/proxy/proxymetrics.go
View file

@ -1,74 +0,0 @@
package proxy
import (
"expvar"
"sync/atomic"
)
// Metrics is used to hold metric counters
// related to the proxy
type Metrics struct {
Requests uint64
Hits uint64
Misses uint64
BytesPulled uint64
BytesPushed uint64
}
type proxyMetricsCollector struct {
blobMetrics Metrics
manifestMetrics Metrics
}
// BlobPull tracks metrics about blobs pulled into the cache
func (pmc *proxyMetricsCollector) BlobPull(bytesPulled uint64) {
atomic.AddUint64(&pmc.blobMetrics.Misses, 1)
atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytesPulled)
}
// BlobPush tracks metrics about blobs pushed to clients
func (pmc *proxyMetricsCollector) BlobPush(bytesPushed uint64) {
atomic.AddUint64(&pmc.blobMetrics.Requests, 1)
atomic.AddUint64(&pmc.blobMetrics.Hits, 1)
atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytesPushed)
}
// ManifestPull tracks metrics related to Manifests pulled into the cache
func (pmc *proxyMetricsCollector) ManifestPull(bytesPulled uint64) {
atomic.AddUint64(&pmc.manifestMetrics.Misses, 1)
atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytesPulled)
}
// ManifestPush tracks metrics about manifests pushed to clients
func (pmc *proxyMetricsCollector) ManifestPush(bytesPushed uint64) {
atomic.AddUint64(&pmc.manifestMetrics.Requests, 1)
atomic.AddUint64(&pmc.manifestMetrics.Hits, 1)
atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytesPushed)
}
// proxyMetrics tracks metrics about the proxy cache. This is
// kept globally and made available via expvar.
var proxyMetrics = &proxyMetricsCollector{}
func init() {
registry := expvar.Get("registry")
if registry == nil {
registry = expvar.NewMap("registry")
}
pm := registry.(*expvar.Map).Get("proxy")
if pm == nil {
pm = &expvar.Map{}
pm.(*expvar.Map).Init()
registry.(*expvar.Map).Set("proxy", pm)
}
pm.(*expvar.Map).Set("blobs", expvar.Func(func() interface{} {
return proxyMetrics.blobMetrics
}))
pm.(*expvar.Map).Set("manifests", expvar.Func(func() interface{} {
return proxyMetrics.manifestMetrics
}))
}

248
docs/proxy/proxyregistry.go
View file

@ -1,248 +0,0 @@
package proxy
import (
"fmt"
"net/http"
"net/url"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/context"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/distribution/registry/proxy/scheduler"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/driver"
)
// proxyingRegistry fetches content from a remote registry and caches it locally
type proxyingRegistry struct {
embedded distribution.Namespace // provides local registry functionality
scheduler *scheduler.TTLExpirationScheduler
remoteURL url.URL
authChallenger authChallenger
}
// NewRegistryPullThroughCache creates a registry acting as a pull through cache
func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Namespace, driver driver.StorageDriver, config configuration.Proxy) (distribution.Namespace, error) {
remoteURL, err := url.Parse(config.RemoteURL)
if err != nil {
return nil, err
}
v := storage.NewVacuum(ctx, driver)
s := scheduler.New(ctx, driver, "/scheduler-state.json")
s.OnBlobExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
if err != nil {
return err
}
blobs := repo.Blobs(ctx)
// Clear the repository reference and descriptor caches
err = blobs.Delete(ctx, r.Digest())
if err != nil {
return err
}
err = v.RemoveBlob(r.Digest().String())
if err != nil {
return err
}
return nil
})
s.OnManifestExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
if err != nil {
return err
}
manifests, err := repo.Manifests(ctx)
if err != nil {
return err
}
err = manifests.Delete(ctx, r.Digest())
if err != nil {
return err
}
return nil
})
err = s.Start()
if err != nil {
return nil, err
}
cs, err := configureAuth(config.Username, config.Password)
if err != nil {
return nil, err
}
return &proxyingRegistry{
embedded: registry,
scheduler: s,
remoteURL: *remoteURL,
authChallenger: &remoteAuthChallenger{
remoteURL: *remoteURL,
cm: auth.NewSimpleChallengeManager(),
cs: cs,
},
}, nil
}
func (pr *proxyingRegistry) Scope() distribution.Scope {
return distribution.GlobalScope
}
func (pr *proxyingRegistry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) {
return pr.embedded.Repositories(ctx, repos, last)
}
func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named) (distribution.Repository, error) {
c := pr.authChallenger
tr := transport.NewTransport(http.DefaultTransport,
auth.NewAuthorizer(c.challengeManager(), auth.NewTokenHandler(http.DefaultTransport, c.credentialStore(), name.Name(), "pull")))
localRepo, err := pr.embedded.Repository(ctx, name)
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification())
if err != nil {
return nil, err
}
remoteRepo, err := client.NewRepository(ctx, name, pr.remoteURL.String(), tr)
if err != nil {
return nil, err
}
remoteManifests, err := remoteRepo.Manifests(ctx)
if err != nil {
return nil, err
}
return &proxiedRepository{
blobStore: &proxyBlobStore{
localStore: localRepo.Blobs(ctx),
remoteStore: remoteRepo.Blobs(ctx),
scheduler: pr.scheduler,
repositoryName: name,
authChallenger: pr.authChallenger,
},
manifests: &proxyManifestStore{
repositoryName: name,
localManifests: localManifests, // Options?
remoteManifests: remoteManifests,
ctx: ctx,
scheduler: pr.scheduler,
authChallenger: pr.authChallenger,
},
name: name,
tags: &proxyTagService{
localTags: localRepo.Tags(ctx),
remoteTags: remoteRepo.Tags(ctx),
authChallenger: pr.authChallenger,
},
}, nil
}
func (pr *proxyingRegistry) Blobs() distribution.BlobEnumerator {
return pr.embedded.Blobs()
}
func (pr *proxyingRegistry) BlobStatter() distribution.BlobStatter {
return pr.embedded.BlobStatter()
}
// authChallenger encapsulates a request to the upstream to establish credential challenges
type authChallenger interface {
tryEstablishChallenges(context.Context) error
challengeManager() auth.ChallengeManager
credentialStore() auth.CredentialStore
}
type remoteAuthChallenger struct {
remoteURL url.URL
sync.Mutex
cm auth.ChallengeManager
cs auth.CredentialStore
}
func (r *remoteAuthChallenger) credentialStore() auth.CredentialStore {
return r.cs
}
func (r *remoteAuthChallenger) challengeManager() auth.ChallengeManager {
return r.cm
}
// tryEstablishChallenges will attempt to get a challenge type for the upstream if none currently exist
func (r *remoteAuthChallenger) tryEstablishChallenges(ctx context.Context) error {
r.Lock()
defer r.Unlock()
remoteURL := r.remoteURL
remoteURL.Path = "/v2/"
challenges, err := r.cm.GetChallenges(r.remoteURL)
if err != nil {
return err
}
if len(challenges) > 0 {
return nil
}
// establish challenge type with upstream
if err := ping(r.cm, remoteURL.String(), challengeHeader); err != nil {
return err
}
context.GetLogger(ctx).Infof("Challenge established with upstream : %s %s", remoteURL, r.cm)
return nil
}
// proxiedRepository uses proxying blob and manifest services to serve content
// locally, or pulling it through from a remote and caching it locally if it doesn't
// already exist
type proxiedRepository struct {
blobStore distribution.BlobStore
manifests distribution.ManifestService
name reference.Named
tags distribution.TagService
}
func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
return pr.manifests, nil
}
func (pr *proxiedRepository) Blobs(ctx context.Context) distribution.BlobStore {
return pr.blobStore
}
func (pr *proxiedRepository) Named() reference.Named {
return pr.name
}
func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService {
return pr.tags
}

65
docs/proxy/proxytagservice.go
View file

@ -1,65 +0,0 @@
package proxy
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
// proxyTagService supports local and remote lookup of tags.
type proxyTagService struct {
localTags distribution.TagService
remoteTags distribution.TagService
authChallenger authChallenger
}
var _ distribution.TagService = proxyTagService{}
// Get attempts to get the most recent digest for the tag by checking the remote
// tag service first and then caching it locally. If the remote is unavailable
// the local association is returned
func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
err := pt.authChallenger.tryEstablishChallenges(ctx)
if err == nil {
desc, err := pt.remoteTags.Get(ctx, tag)
if err == nil {
err := pt.localTags.Tag(ctx, tag, desc)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
}
desc, err := pt.localTags.Get(ctx, tag)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
return distribution.ErrUnsupported
}
func (pt proxyTagService) Untag(ctx context.Context, tag string) error {
err := pt.localTags.Untag(ctx, tag)
if err != nil {
return err
}
return nil
}
func (pt proxyTagService) All(ctx context.Context) ([]string, error) {
err := pt.authChallenger.tryEstablishChallenges(ctx)
if err == nil {
tags, err := pt.remoteTags.All(ctx)
if err == nil {
return tags, err
}
}
return pt.localTags.All(ctx)
}
func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
return []string{}, distribution.ErrUnsupported
}

182
docs/proxy/proxytagservice_test.go
View file

@ -1,182 +0,0 @@
package proxy
import (
"reflect"
"sort"
"sync"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
type mockTagStore struct {
mapping map[string]distribution.Descriptor
sync.Mutex
}
var _ distribution.TagService = &mockTagStore{}
func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
m.Lock()
defer m.Unlock()
if d, ok := m.mapping[tag]; ok {
return d, nil
}
return distribution.Descriptor{}, distribution.ErrTagUnknown{}
}
func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
m.Lock()
defer m.Unlock()
m.mapping[tag] = desc
return nil
}
func (m *mockTagStore) Untag(ctx context.Context, tag string) error {
m.Lock()
defer m.Unlock()
if _, ok := m.mapping[tag]; ok {
delete(m.mapping, tag)
return nil
}
return distribution.ErrTagUnknown{}
}
func (m *mockTagStore) All(ctx context.Context) ([]string, error) {
m.Lock()
defer m.Unlock()
var tags []string
for tag := range m.mapping {
tags = append(tags, tag)
}
return tags, nil
}
func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService {
if local == nil {
local = make(map[string]distribution.Descriptor)
}
if remote == nil {
remote = make(map[string]distribution.Descriptor)
}
return &proxyTagService{
localTags: &mockTagStore{mapping: local},
remoteTags: &mockTagStore{mapping: remote},
authChallenger: &mockChallenger{},
}
}
func TestGet(t *testing.T) {
remoteDesc := distribution.Descriptor{Size: 42}
remoteTag := "remote"
proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil)
ctx := context.Background()
// Get pre-loaded tag
d, err := proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if proxyTags.authChallenger.(*mockChallenger).count != 1 {
t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger)
}
if !reflect.DeepEqual(d, remoteDesc) {
t.Fatal("unable to get put tag")
}
local, err := proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
if !reflect.DeepEqual(local, remoteDesc) {
t.Fatalf("unexpected descriptor pulled through")
}
// Manually overwrite remote tag
newRemoteDesc := distribution.Descriptor{Size: 43}
err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc)
if err != nil {
t.Fatal(err)
}
d, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if proxyTags.authChallenger.(*mockChallenger).count != 2 {
t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger)
}
if !reflect.DeepEqual(d, newRemoteDesc) {
t.Fatal("unable to get put tag")
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
// untag, ensure it's removed locally, but present in remote
err = proxyTags.Untag(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err == nil {
t.Fatalf("Expected error getting Untag'd tag")
}
_, err = proxyTags.remoteTags.Get(ctx, remoteTag)
if err != nil {
t.Fatalf("remote tag should not be untagged with proxyTag.Untag")
}
_, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("untagged tag should be pulled through")
}
if proxyTags.authChallenger.(*mockChallenger).count != 3 {
t.Fatalf("Expected 3 auth challenge calls, got %#v", proxyTags.authChallenger)
}
// Add another tag. Ensure both tags appear in 'All'
err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42})
if err != nil {
t.Fatal(err)
}
all, err := proxyTags.All(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("Unexpected tag length returned from All() : %d ", len(all))
}
sort.Strings(all)
if all[0] != "funtag" && all[1] != "remote" {
t.Fatalf("Unexpected tags returned from All() : %v ", all)
}
if proxyTags.authChallenger.(*mockChallenger).count != 4 {
t.Fatalf("Expected 4 auth challenge calls, got %#v", proxyTags.authChallenger)
}
}

258
docs/proxy/scheduler/scheduler.go
View file

@ -1,258 +0,0 @@
package scheduler
import (
"encoding/json"
"fmt"
"sync"
"time"
"github.com/docker/distribution/context"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/storage/driver"
)
// onTTLExpiryFunc is called when a repository's TTL expires
type expiryFunc func(reference.Reference) error
const (
entryTypeBlob = iota
entryTypeManifest
indexSaveFrequency = 5 * time.Second
)
// schedulerEntry represents an entry in the scheduler
// fields are exported for serialization
type schedulerEntry struct {
Key string `json:"Key"`
Expiry time.Time `json:"ExpiryData"`
EntryType int `json:"EntryType"`
timer *time.Timer
}
// New returns a new instance of the scheduler
func New(ctx context.Context, driver driver.StorageDriver, path string) *TTLExpirationScheduler {
return &TTLExpirationScheduler{
entries: make(map[string]*schedulerEntry),
driver: driver,
pathToStateFile: path,
ctx: ctx,
stopped: true,
doneChan: make(chan struct{}),
saveTimer: time.NewTicker(indexSaveFrequency),
}
}
// TTLExpirationScheduler is a scheduler used to perform actions
// when TTLs expire
type TTLExpirationScheduler struct {
sync.Mutex
entries map[string]*schedulerEntry
driver driver.StorageDriver
ctx context.Context
pathToStateFile string
stopped bool
onBlobExpire expiryFunc
onManifestExpire expiryFunc
indexDirty bool
saveTimer *time.Ticker
doneChan chan struct{}
}
// OnBlobExpire is called when a scheduled blob's TTL expires
func (ttles *TTLExpirationScheduler) OnBlobExpire(f expiryFunc) {
ttles.Lock()
defer ttles.Unlock()
ttles.onBlobExpire = f
}
// OnManifestExpire is called when a scheduled manifest's TTL expires
func (ttles *TTLExpirationScheduler) OnManifestExpire(f expiryFunc) {
ttles.Lock()
defer ttles.Unlock()
ttles.onManifestExpire = f
}
// AddBlob schedules a blob cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddBlob(blobRef reference.Canonical, ttl time.Duration) error {
ttles.Lock()
defer ttles.Unlock()
if ttles.stopped {
return fmt.Errorf("scheduler not started")
}
ttles.add(blobRef, ttl, entryTypeBlob)
return nil
}
// AddManifest schedules a manifest cleanup after ttl expires
func (ttles *TTLExpirationScheduler) AddManifest(manifestRef reference.Canonical, ttl time.Duration) error {
ttles.Lock()
defer ttles.Unlock()
if ttles.stopped {
return fmt.Errorf("scheduler not started")
}
ttles.add(manifestRef, ttl, entryTypeManifest)
return nil
}
// Start starts the scheduler
func (ttles *TTLExpirationScheduler) Start() error {
ttles.Lock()
defer ttles.Unlock()
err := ttles.readState()
if err != nil {
return err
}
if !ttles.stopped {
return fmt.Errorf("Scheduler already started")
}
context.GetLogger(ttles.ctx).Infof("Starting cached object TTL expiration scheduler...")
ttles.stopped = false
// Start timer for each deserialized entry
for _, entry := range ttles.entries {
entry.timer = ttles.startTimer(entry, entry.Expiry.Sub(time.Now()))
}
// Start a ticker to periodically save the entries index
go func() {
for {
select {
case <-ttles.saveTimer.C:
if !ttles.indexDirty {
continue
}
ttles.Lock()
err := ttles.writeState()
if err != nil {
context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
} else {
ttles.indexDirty = false
}
ttles.Unlock()
case <-ttles.doneChan:
return
}
}
}()
return nil
}
func (ttles *TTLExpirationScheduler) add(r reference.Reference, ttl time.Duration, eType int) {
entry := &schedulerEntry{
Key: r.String(),
Expiry: time.Now().Add(ttl),
EntryType: eType,
}
context.GetLogger(ttles.ctx).Infof("Adding new scheduler entry for %s with ttl=%s", entry.Key, entry.Expiry.Sub(time.Now()))
if oldEntry, present := ttles.entries[entry.Key]; present && oldEntry.timer != nil {
oldEntry.timer.Stop()
}
ttles.entries[entry.Key] = entry
entry.timer = ttles.startTimer(entry, ttl)
ttles.indexDirty = true
}
func (ttles *TTLExpirationScheduler) startTimer(entry *schedulerEntry, ttl time.Duration) *time.Timer {
return time.AfterFunc(ttl, func() {
ttles.Lock()
defer ttles.Unlock()
var f expiryFunc
switch entry.EntryType {
case entryTypeBlob:
f = ttles.onBlobExpire
case entryTypeManifest:
f = ttles.onManifestExpire
default:
f = func(reference.Reference) error {
return fmt.Errorf("scheduler entry type")
}
}
ref, err := reference.Parse(entry.Key)
if err == nil {
if err := f(ref); err != nil {
context.GetLogger(ttles.ctx).Errorf("Scheduler error returned from OnExpire(%s): %s", entry.Key, err)
}
} else {
context.GetLogger(ttles.ctx).Errorf("Error unpacking reference: %s", err)
}
delete(ttles.entries, entry.Key)
ttles.indexDirty = true
})
}
// Stop stops the scheduler.
func (ttles *TTLExpirationScheduler) Stop() {
ttles.Lock()
defer ttles.Unlock()
if err := ttles.writeState(); err != nil {
context.GetLogger(ttles.ctx).Errorf("Error writing scheduler state: %s", err)
}
for _, entry := range ttles.entries {
entry.timer.Stop()
}
close(ttles.doneChan)
ttles.saveTimer.Stop()
ttles.stopped = true
}
func (ttles *TTLExpirationScheduler) writeState() error {
jsonBytes, err := json.Marshal(ttles.entries)
if err != nil {
return err
}
err = ttles.driver.PutContent(ttles.ctx, ttles.pathToStateFile, jsonBytes)
if err != nil {
return err
}
return nil
}
func (ttles *TTLExpirationScheduler) readState() error {
if _, err := ttles.driver.Stat(ttles.ctx, ttles.pathToStateFile); err != nil {
switch err := err.(type) {
case driver.PathNotFoundError:
return nil
default:
return err
}
}
bytes, err := ttles.driver.GetContent(ttles.ctx, ttles.pathToStateFile)
if err != nil {
return err
}
err = json.Unmarshal(bytes, &ttles.entries)
if err != nil {
return err
}
return nil
}

188
docs/proxy/scheduler/scheduler_test.go
View file

@ -1,188 +0,0 @@
package scheduler
import (
"encoding/json"
"testing"
"time"
"github.com/docker/distribution/context"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
func testRefs(t *testing.T) (reference.Reference, reference.Reference, reference.Reference) {
ref1, err := reference.Parse("testrepo@sha256:aaaaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
ref2, err := reference.Parse("testrepo@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
ref3, err := reference.Parse("testrepo@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
return ref1, ref2, ref3
}
func TestSchedule(t *testing.T) {
ref1, ref2, ref3 := testRefs(t)
timeUnit := time.Millisecond
remainingRepos := map[string]bool{
ref1.String(): true,
ref2.String(): true,
ref3.String(): true,
}
s := New(context.Background(), inmemory.New(), "/ttl")
deleteFunc := func(repoName reference.Reference) error {
if len(remainingRepos) == 0 {
t.Fatalf("Incorrect expiry count")
}
_, ok := remainingRepos[repoName.String()]
if !ok {
t.Fatalf("Trying to remove nonexistent repo: %s", repoName)
}
t.Log("removing", repoName)
delete(remainingRepos, repoName.String())
return nil
}
s.onBlobExpire = deleteFunc
err := s.Start()
if err != nil {
t.Fatalf("Error starting ttlExpirationScheduler: %s", err)
}
s.add(ref1, 3*timeUnit, entryTypeBlob)
s.add(ref2, 1*timeUnit, entryTypeBlob)
func() {
s.add(ref3, 1*timeUnit, entryTypeBlob)
}()
// Ensure all repos are deleted
<-time.After(50 * timeUnit)
if len(remainingRepos) != 0 {
t.Fatalf("Repositories remaining: %#v", remainingRepos)
}
}
func TestRestoreOld(t *testing.T) {
ref1, ref2, _ := testRefs(t)
remainingRepos := map[string]bool{
ref1.String(): true,
ref2.String(): true,
}
deleteFunc := func(r reference.Reference) error {
if r.String() == ref1.String() && len(remainingRepos) == 2 {
t.Errorf("ref1 should be removed first")
}
_, ok := remainingRepos[r.String()]
if !ok {
t.Fatalf("Trying to remove nonexistent repo: %s", r)
}
delete(remainingRepos, r.String())
return nil
}
timeUnit := time.Millisecond
serialized, err := json.Marshal(&map[string]schedulerEntry{
ref1.String(): {
Expiry: time.Now().Add(1 * timeUnit),
Key: ref1.String(),
EntryType: 0,
},
ref2.String(): {
Expiry: time.Now().Add(-3 * timeUnit), // TTL passed, should be removed first
Key: ref2.String(),
EntryType: 0,
},
})
if err != nil {
t.Fatalf("Error serializing test data: %s", err.Error())
}
ctx := context.Background()
pathToStatFile := "/ttl"
fs := inmemory.New()
err = fs.PutContent(ctx, pathToStatFile, serialized)
if err != nil {
t.Fatal("Unable to write serialized data to fs")
}
s := New(context.Background(), fs, "/ttl")
s.onBlobExpire = deleteFunc
err = s.Start()
if err != nil {
t.Fatalf("Error starting ttlExpirationScheduler: %s", err)
}
<-time.After(50 * timeUnit)
if len(remainingRepos) != 0 {
t.Fatalf("Repositories remaining: %#v", remainingRepos)
}
}
func TestStopRestore(t *testing.T) {
ref1, ref2, _ := testRefs(t)
timeUnit := time.Millisecond
remainingRepos := map[string]bool{
ref1.String(): true,
ref2.String(): true,
}
deleteFunc := func(r reference.Reference) error {
delete(remainingRepos, r.String())
return nil
}
fs := inmemory.New()
pathToStateFile := "/ttl"
s := New(context.Background(), fs, pathToStateFile)
s.onBlobExpire = deleteFunc
err := s.Start()
if err != nil {
t.Fatalf(err.Error())
}
s.add(ref1, 300*timeUnit, entryTypeBlob)
s.add(ref2, 100*timeUnit, entryTypeBlob)
// Start and stop before all operations complete
// state will be written to fs
s.Stop()
time.Sleep(10 * time.Millisecond)
// v2 will restore state from fs
s2 := New(context.Background(), fs, pathToStateFile)
s2.onBlobExpire = deleteFunc
err = s2.Start()
if err != nil {
t.Fatalf("Error starting v2: %s", err.Error())
}
<-time.After(500 * timeUnit)
if len(remainingRepos) != 0 {
t.Fatalf("Repositories remaining: %#v", remainingRepos)
}
}
func TestDoubleStart(t *testing.T) {
s := New(context.Background(), inmemory.New(), "/ttl")
err := s.Start()
if err != nil {
t.Fatalf("Unable to start scheduler")
}
err = s.Start()
if err == nil {
t.Fatalf("Scheduler started twice without error")
}
}

215
docs/recipes/apache.md Normal file
View file

@ -0,0 +1,215 @@
<!--[metadata]>
+++
title = "Authenticating proxy with apache"
description = "Restricting access to your registry using an apache proxy"
keywords = ["registry, on-prem, images, tags, repository, distribution, authentication, proxy, apache, httpd, TLS, recipe, advanced"]
[menu.main]
parent="smn_recipes"
+++
<![end-metadata]-->
# Authenticating proxy with apache
## Use-case
People already relying on an apache proxy to authenticate their users to other services might want to leverage it and have Registry communications tunneled through the same pipeline.
Usually, that includes enterprise setups using LDAP/AD on the backend and a SSO mechanism fronting their internal http portal.
### Alternatives
If you just want authentication for your registry, and are happy maintaining users access separately, you should really consider sticking with the native [basic auth registry feature](../deploying.md#native-basic-auth).
### Solution
With the method presented here, you implement basic authentication for docker engines in a reverse proxy that sits in front of your registry.
While we use a simple htpasswd file as an example, any other apache authentication backend should be fairly easy to implement once you are done with the example.
We also implement push restriction (to a limited user group) for the sake of the example. Again, you should modify this to fit your mileage.
### Gotchas
While this model gives you the ability to use whatever authentication backend you want through the secondary authentication mechanism implemented inside your proxy, it also requires that you move TLS termination from the Registry to the proxy itself.
Furthermore, introducing an extra http layer in your communication pipeline will make it more complex to deploy, maintain, and debug, and will possibly create issues.
## Setting things up
Read again [the requirements](index.md#requirements).
Ready?
Run the following script:
```
mkdir -p auth
mkdir -p data
# This is the main apache configuration you will use
cat <<EOF > auth/httpd.conf
LoadModule headers_module modules/mod_headers.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule unixd_module modules/mod_unixd.so
<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
<IfModule unixd_module>
User daemon
Group daemon
</IfModule>
ServerAdmin you@example.com
ErrorLog /proc/self/fd/2
LogLevel warn
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
CustomLog /proc/self/fd/1 common
</IfModule>
ServerRoot "/usr/local/apache2"
Listen 5043
<Directory />
AllowOverride none
Require all denied
</Directory>
<VirtualHost *:5043>
ServerName myregistrydomain.com
SSLEngine on
SSLCertificateFile /usr/local/apache2/conf/domain.crt
SSLCertificateKeyFile /usr/local/apache2/conf/domain.key
## SSL settings recommandation from: https://raymii.org/s/tutorials/Strong_SSL_Security_On_Apache2.html
# Anti CRIME
SSLCompression off
# POODLE and other stuff
SSLProtocol all -SSLv2 -SSLv3 -TLSv1
# Secure cypher suites
SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
SSLHonorCipherOrder on
Header always set "Docker-Distribution-Api-Version" "registry/2.0"
Header onsuccess set "Docker-Distribution-Api-Version" "registry/2.0"
RequestHeader set X-Forwarded-Proto "https"
ProxyRequests off
ProxyPreserveHost on
# no proxy for /error/ (Apache HTTPd errors messages)
ProxyPass /error/ !
ProxyPass /v2 http://registry:5000/v2
ProxyPassReverse /v2 http://registry:5000/v2
<Location /v2>
Order deny,allow
Allow from all
AuthName "Registry Authentication"
AuthType basic
AuthUserFile "/usr/local/apache2/conf/httpd.htpasswd"
AuthGroupFile "/usr/local/apache2/conf/httpd.groups"
# Read access to authentified users
<Limit GET HEAD>
Require valid-user
</Limit>
# Write access to docker-deployer only
<Limit POST PUT DELETE PATCH>
Require group pusher
</Limit>
</Location>
</VirtualHost>
EOF
# Now, create a password file for "testuser" and "testpassword"
docker run --entrypoint htpasswd httpd:2.4 -Bbn testuser testpassword > auth/httpd.htpasswd
# Create another one for "testuserpush" and "testpasswordpush"
docker run --entrypoint htpasswd httpd:2.4 -Bbn testuserpush testpasswordpush >> auth/httpd.htpasswd
# Create your group file
echo "pusher: testuserpush" > auth/httpd.groups
# Copy over your certificate files
cp domain.crt auth
cp domain.key auth
# Now create your compose file
cat <<EOF > docker-compose.yml
apache:
image: "httpd:2.4"
hostname: myregistrydomain.com
ports:
- 5043:5043
links:
- registry:registry
volumes:
- `pwd`/auth:/usr/local/apache2/conf
registry:
image: registry:2
ports:
- 127.0.0.1:5000:5000
volumes:
- `pwd`/data:/var/lib/registry
EOF
```
## Starting and stopping
Now, start your stack:
docker-compose up -d
Login with a "push" authorized user (using `testuserpush` and `testpasswordpush`), then tag and push your first image:
docker login myregistrydomain.com:5043
docker tag ubuntu myregistrydomain.com:5043/test
docker push myregistrydomain.com:5043/test
Now, login with a "pull-only" user (using `testuser` and `testpassword`), then pull back the image:
docker login myregistrydomain.com:5043
docker pull myregistrydomain.com:5043/test
Verify that the "pull-only" can NOT push:
docker push myregistrydomain.com:5043/test

37
docs/recipes/index.md Normal file
View file

@ -0,0 +1,37 @@
<!--[metadata]>
+++
title = "Recipes Overview"
description = "Fun stuff to do with your registry"
keywords = ["registry, on-prem, images, tags, repository, distribution, recipes, advanced"]
[menu.main]
parent="smn_recipes"
weight=-10
+++
<![end-metadata]-->
# Recipes
You will find here a list of "recipes", end-to-end scenarios for exotic or otherwise advanced use-cases.
Most users are not expected to have a use for these.
## Requirements
You should have followed entirely the basic [deployment guide](../deploying.md).
If you have not, please take the time to do so.
At this point, it's assumed that:
* you understand Docker security requirements, and how to configure your docker engines properly
* you have installed Docker Compose
* it's HIGHLY recommended that you get a certificate from a known CA instead of self-signed certificates
* inside the current directory, you have a X509 `domain.crt` and `domain.key`, for the CN `myregistrydomain.com`
* be sure you have stopped and removed any previously running registry (typically `docker stop registry && docker rm -v registry`)
## The List
* [using Apache as an authenticating proxy](apache.md)
* [using Nginx as an authenticating proxy](nginx.md)
* [running a Registry on OS X](osx-setup-guide.md)
* [mirror the Docker Hub](mirror.md)

21
docs/recipes/menu.md Normal file
View file

@ -0,0 +1,21 @@
<!--[metadata]>
+++
title = "Recipes"
description = "Registry Recipes"
keywords = ["registry, on-prem, images, tags, repository, distribution"]
type = "menu"
[menu.main]
identifier="smn_recipes"
parent="smn_registry"
weight=6
+++
<![end-metadata]-->
# Recipes
## The List
* [using Apache as an authenticating proxy](apache.md)
* [using Nginx as an authenticating proxy](nginx.md)
* [running a Registry on OS X](osx-setup-guide.md)
* [mirror the Docker Hub](mirror.md)

74
docs/recipes/mirror.md Normal file
View file

@ -0,0 +1,74 @@
<!--[metadata]>
+++
title = "Mirroring Docker Hub"
description = "Setting-up a local mirror for Docker Hub images"
keywords = ["registry, on-prem, images, tags, repository, distribution, mirror, Hub, recipe, advanced"]
[menu.main]
parent="smn_recipes"
+++
<![end-metadata]-->
# Registry as a pull through cache
## Use-case
If you have multiple instances of Docker running in your environment (e.g., multiple physical or virtual machines, all running the Docker daemon), each time one of them requires an image that it doesn’t have it will go out to the internet and fetch it from the public Docker registry. By running a local registry mirror, you can keep most of the redundant image fetch traffic on your local network.
### Alternatives
Alternatively, if the set of images you are using is well delimited, you can simply pull them manually and push them to a simple, local, private registry.
Furthermore, if your images are all built in-house, not using the Hub at all and relying entirely on your local registry is the simplest scenario.
### Gotcha
It's currently not possible to mirror another private registry. Only the central Hub can be mirrored.
### Solution
The Registry can be configured as a pull through cache. In this mode a Registry responds to all normal docker pull requests but stores all content locally.
## How does it work?
The first time you request an image from your local registry mirror, it pulls the image from the public Docker registry and stores it locally before handing it back to you. On subsequent requests, the local registry mirror is able to serve the image from its own storage.
### What if the content changes on the Hub?
When a pull is attempted with a tag, the Registry will check the remote to ensure if it has the latest version of the requested content. If it doesn't it will fetch the latest content and cache it.
### What about my disk?
In environments with high churn rates, stale data can build up in the cache. When running as a pull through cache the Registry will periodically remove old content to save disk space. Subsequent requests for removed content will cause a remote fetch and local re-caching.
To ensure best performance and guarantee correctness the Registry cache should be configured to use the `filesystem` driver for storage.
## Running a Registry as a pull through cache
The easiest way to run a registry as a pull through cache is to run the official Registry image.
Multiple registry caches can be deployed over the same back-end. A single registry cache will ensure that concurrent requests do not pull duplicate data, but this property will not hold true for a registry cache cluster.
### Configuring the cache
To configure a Registry to run as a pull through cache, the addition of a `proxy` section is required to the config file.
In order to access private images on the Docker Hub, a username and password can be supplied.
proxy:
remoteurl: https://registry-1.docker.io
username: [username]
password: [password]
> :warn: if you specify a username and password, it's very important to understand that private resources that this user has access to on the Hub will be made available on your mirror. It's thus paramount that you secure your mirror by implementing authentication if you expect these resources to stay private!
### Configuring the Docker daemon
You will need to pass the `--registry-mirror` option to your Docker daemon on startup:
docker --registry-mirror=https://<my-docker-mirror-host> daemon
For example, if your mirror is serving on `http://10.0.0.2:5000`, you would run:
docker --registry-mirror=https://10.0.0.2:5000 daemon
NOTE: Depending on your local host setup, you may be able to add the `--registry-mirror` option to the `DOCKER_OPTS` variable in `/etc/default/docker`.

190
docs/recipes/nginx.md Normal file
View file

@ -0,0 +1,190 @@
<!--[metadata]>
+++
title = "Authenticating proxy with nginx"
description = "Restricting access to your registry using a nginx proxy"
keywords = ["registry, on-prem, images, tags, repository, distribution, nginx, proxy, authentication, TLS, recipe, advanced"]
[menu.main]
parent="smn_recipes"
+++
<![end-metadata]-->
# Authenticating proxy with nginx
## Use-case
People already relying on a nginx proxy to authenticate their users to other services might want to leverage it and have Registry communications tunneled through the same pipeline.
Usually, that includes enterprise setups using LDAP/AD on the backend and a SSO mechanism fronting their internal http portal.
### Alternatives
If you just want authentication for your registry, and are happy maintaining users access separately, you should really consider sticking with the native [basic auth registry feature](../deploying.md#native-basic-auth).
### Solution
With the method presented here, you implement basic authentication for docker engines in a reverse proxy that sits in front of your registry.
While we use a simple htpasswd file as an example, any other nginx authentication backend should be fairly easy to implement once you are done with the example.
We also implement push restriction (to a limited user group) for the sake of the example. Again, you should modify this to fit your mileage.
### Gotchas
While this model gives you the ability to use whatever authentication backend you want through the secondary authentication mechanism implemented inside your proxy, it also requires that you move TLS termination from the Registry to the proxy itself.
Furthermore, introducing an extra http layer in your communication pipeline will make it more complex to deploy, maintain, and debug, and will possibly create issues. Make sure the extra complexity is required.
For instance, Amazon's Elastic Load Balancer (ELB) in HTTPS mode already sets the following client header:
```
X-Real-IP
X-Forwarded-For
X-Forwarded-Proto
```
So if you have an nginx sitting behind it, should remove these lines from the example config below:
```
X-Real-IP $remote_addr; # pass on real client's IP
X-Forwarded-For $proxy_add_x_forwarded_for;
X-Forwarded-Proto $scheme;
```
Otherwise nginx will reset the ELB's values, and the requests will not be routed properly. For more information, see [#970](https://github.com/docker/distribution/issues/970).
## Setting things up
Read again [the requirements](index.md#requirements).
Ready?
--
Create the required directories
```
mkdir -p auth
mkdir -p data
```
Create the main nginx configuration you will use.
```
cat <<EOF > auth/nginx.conf
events {
worker_connections 1024;
}
http {
upstream docker-registry {
server registry:5000;
}
## Set a variable to help us decide if we need to add the
## 'Docker-Distribution-Api-Version' header.
## The registry always sets this header.
## In the case of nginx performing auth, the header will be unset
## since nginx is auth-ing before proxying.
map \$upstream_http_docker_distribution_api_version \$docker_distribution_api_version {
'registry/2.0' '';
default registry/2.0;
}
server {
listen 443 ssl;
server_name myregistrydomain.com;
# SSL
ssl_certificate /etc/nginx/conf.d/domain.crt;
ssl_certificate_key /etc/nginx/conf.d/domain.key;
# Recommendations from https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
ssl_protocols TLSv1.1 TLSv1.2;
ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
# disable any limits to avoid HTTP 413 for large image uploads
client_max_body_size 0;
# required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486)
chunked_transfer_encoding on;
location /v2/ {
# Do not allow connections from docker 1.5 and earlier
# docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents
if (\$http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*\$" ) {
return 404;
}
# To add basic authentication to v2 use auth_basic setting.
auth_basic "Registry realm";
auth_basic_user_file /etc/nginx/conf.d/nginx.htpasswd;
## If $docker_distribution_api_version is empty, the header will not be added.
## See the map directive above where this variable is defined.
add_header 'Docker-Distribution-Api-Version' \$docker_distribution_api_version always;
proxy_pass http://docker-registry;
proxy_set_header Host \$http_host; # required for docker client's sake
proxy_set_header X-Real-IP \$remote_addr; # pass on real client's IP
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_read_timeout 900;
}
}
}
EOF
```
Now create a password file for "testuser" and "testpassword"
```
docker run --rm --entrypoint htpasswd registry:2 -bn testuser testpassword > auth/nginx.htpasswd
```
Copy over your certificate files
```
cp domain.crt auth
cp domain.key auth
```
Now create your compose file
```
cat <<EOF > docker-compose.yml
nginx:
image: "nginx:1.9"
ports:
- 5043:443
links:
- registry:registry
volumes:
- ./auth:/etc/nginx/conf.d
- ./auth/nginx.conf:/etc/nginx/nginx.conf:ro
registry:
image: registry:2
ports:
- 127.0.0.1:5000:5000
volumes:
- `pwd`./data:/var/lib/registry
EOF
```
## Starting and stopping
Now, start your stack:
docker-compose up -d
Login with a "push" authorized user (using `testuser` and `testpassword`), then tag and push your first image:
docker login -u=testuser -p=testpassword -e=root@example.ch myregistrydomain.com:5043
docker tag ubuntu myregistrydomain.com:5043/test
docker push myregistrydomain.com:5043/test
docker pull myregistrydomain.com:5043/test

81
docs/recipes/osx-setup-guide.md Normal file
View file

@ -0,0 +1,81 @@
<!--[metadata]>
+++
title = "Running on OS X"
description = "Explains how to run a registry on OS X"
keywords = ["registry, on-prem, images, tags, repository, distribution, OS X, recipe, advanced"]
[menu.main]
parent="smn_recipes"
+++
<![end-metadata]-->
# OS X Setup Guide
## Use-case
This is useful if you intend to run a registry server natively on OS X.
### Alternatives
You can start a VM on OS X, and deploy your registry normally as a container using Docker inside that VM.
The simplest road to get there is traditionally to use the [docker Toolbox](https://www.docker.com/toolbox), or [docker-machine](/machine/index.md), which usually relies on the [boot2docker](http://boot2docker.io/) iso inside a VirtualBox VM.
### Solution
Using the method described here, you install and compile your own from the git repository and run it as an OS X agent.
### Gotchas
Production services operation on OS X is out of scope of this document. Be sure you understand well these aspects before considering going to production with this.
## Setup golang on your machine
If you know, safely skip to the next section.
If you don't, the TLDR is:
bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
source ~/.gvm/scripts/gvm
gvm install go1.4.2
gvm use go1.4.2
If you want to understand, you should read [How to Write Go Code](https://golang.org/doc/code.html).
## Checkout the Docker Distribution source tree
mkdir -p $GOPATH/src/github.com/docker
git clone https://github.com/docker/distribution.git $GOPATH/src/github.com/docker/distribution
cd $GOPATH/src/github.com/docker/distribution
## Build the binary
GOPATH=$(PWD)/Godeps/_workspace:$GOPATH make binaries
sudo cp bin/registry /usr/local/libexec/registry
## Setup
Copy the registry configuration file in place:
mkdir /Users/Shared/Registry
cp docs/osx/config.yml /Users/Shared/Registry/config.yml
## Running the Docker Registry under launchd
Copy the Docker registry plist into place:
plutil -lint docs/osx/com.docker.registry.plist
cp docs/osx/com.docker.registry.plist ~/Library/LaunchAgents/
chmod 644 ~/Library/LaunchAgents/com.docker.registry.plist
Start the Docker registry:
launchctl load ~/Library/LaunchAgents/com.docker.registry.plist
### Restarting the docker registry service
launchctl stop com.docker.registry
launchctl start com.docker.registry
### Unloading the docker registry service
launchctl unload ~/Library/LaunchAgents/com.docker.registry.plist

42
docs/recipes/osx/com.docker.registry.plist Normal file
View file

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.docker.registry</string>
<key>KeepAlive</key>
<true/>
<key>StandardErrorPath</key>
<string>/Users/Shared/Registry/registry.log</string>
<key>StandardOutPath</key>
<string>/Users/Shared/Registry/registry.log</string>
<key>Program</key>
<string>/usr/local/libexec/registry</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/libexec/registry</string>
<string>/Users/Shared/Registry/config.yml</string>
</array>
<key>Sockets</key>
<dict>
<key>http-listen-address</key>
<dict>
<key>SockServiceName</key>
<string>5000</string>
<key>SockType</key>
<string>dgram</string>
<key>SockFamily</key>
<string>IPv4</string>
</dict>
<key>http-debug-address</key>
<dict>
<key>SockServiceName</key>
<string>5001</string>
<key>SockType</key>
<string>dgram</string>
<key>SockFamily</key>
<string>IPv4</string>
</dict>
</dict>
</dict>
</plist>

16
docs/recipes/osx/config.yml Normal file
View file

@ -0,0 +1,16 @@
version: 0.1
log:
level: info
fields:
service: registry
environment: macbook-air
storage:
cache:
blobdescriptor: inmemory
filesystem:
rootdirectory: /Users/Shared/Registry
http:
addr: 0.0.0.0:5000
secret: mytokensecret
debug:
addr: localhost:5001

345
docs/registry.go
View file

@ -1,345 +0,0 @@
package registry
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net/http"
"os"
"time"
"rsc.io/letsencrypt"
log "github.com/Sirupsen/logrus"
"github.com/Sirupsen/logrus/formatters/logstash"
"github.com/bugsnag/bugsnag-go"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/context"
"github.com/docker/distribution/health"
"github.com/docker/distribution/registry/handlers"
"github.com/docker/distribution/registry/listener"
"github.com/docker/distribution/uuid"
"github.com/docker/distribution/version"
gorhandlers "github.com/gorilla/handlers"
"github.com/spf13/cobra"
"github.com/yvasiyarov/gorelic"
)
// ServeCmd is a cobra command for running the registry.
var ServeCmd = &cobra.Command{
Use: "serve <config>",
Short: "`serve` stores and distributes Docker images",
Long: "`serve` stores and distributes Docker images.",
Run: func(cmd *cobra.Command, args []string) {
// setup context
ctx := context.WithVersion(context.Background(), version.Version)
config, err := resolveConfiguration(args)
if err != nil {
fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
cmd.Usage()
os.Exit(1)
}
if config.HTTP.Debug.Addr != "" {
go func(addr string) {
log.Infof("debug server listening %v", addr)
if err := http.ListenAndServe(addr, nil); err != nil {
log.Fatalf("error listening on debug interface: %v", err)
}
}(config.HTTP.Debug.Addr)
}
registry, err := NewRegistry(ctx, config)
if err != nil {
log.Fatalln(err)
}
if err = registry.ListenAndServe(); err != nil {
log.Fatalln(err)
}
},
}
// A Registry represents a complete instance of the registry.
// TODO(aaronl): It might make sense for Registry to become an interface.
type Registry struct {
config *configuration.Configuration
app *handlers.App
server *http.Server
}
// NewRegistry creates a new registry from a context and configuration struct.
func NewRegistry(ctx context.Context, config *configuration.Configuration) (*Registry, error) {
var err error
ctx, err = configureLogging(ctx, config)
if err != nil {
return nil, fmt.Errorf("error configuring logger: %v", err)
}
// inject a logger into the uuid library. warns us if there is a problem
// with uuid generation under low entropy.
uuid.Loggerf = context.GetLogger(ctx).Warnf
app := handlers.NewApp(ctx, config)
// TODO(aaronl): The global scope of the health checks means NewRegistry
// can only be called once per process.
app.RegisterHealthChecks()
handler := configureReporting(app)
handler = alive("/", handler)
handler = health.Handler(handler)
handler = panicHandler(handler)
handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler)
server := &http.Server{
Handler: handler,
}
return &Registry{
app: app,
config: config,
server: server,
}, nil
}
// ListenAndServe runs the registry's HTTP server.
func (registry *Registry) ListenAndServe() error {
config := registry.config
ln, err := listener.NewListener(config.HTTP.Net, config.HTTP.Addr)
if err != nil {
return err
}
if config.HTTP.TLS.Certificate != "" || config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
tlsConf := &tls.Config{
ClientAuth: tls.NoClientCert,
NextProtos: []string{"http/1.1"},
MinVersion: tls.VersionTLS10,
PreferServerCipherSuites: true,
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
},
}
if config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
if config.HTTP.TLS.Certificate != "" {
return fmt.Errorf("cannot specify both certificate and Let's Encrypt")
}
var m letsencrypt.Manager
if err := m.CacheFile(config.HTTP.TLS.LetsEncrypt.CacheFile); err != nil {
return err
}
if !m.Registered() {
if err := m.Register(config.HTTP.TLS.LetsEncrypt.Email, nil); err != nil {
return err
}
}
tlsConf.GetCertificate = m.GetCertificate
} else {
tlsConf.Certificates = make([]tls.Certificate, 1)
tlsConf.Certificates[0], err = tls.LoadX509KeyPair(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key)
if err != nil {
return err
}
}
if len(config.HTTP.TLS.ClientCAs) != 0 {
pool := x509.NewCertPool()
for _, ca := range config.HTTP.TLS.ClientCAs {
caPem, err := ioutil.ReadFile(ca)
if err != nil {
return err
}
if ok := pool.AppendCertsFromPEM(caPem); !ok {
return fmt.Errorf("Could not add CA to pool")
}
}
for _, subj := range pool.Subjects() {
context.GetLogger(registry.app).Debugf("CA Subject: %s", string(subj))
}
tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
tlsConf.ClientCAs = pool
}
ln = tls.NewListener(ln, tlsConf)
context.GetLogger(registry.app).Infof("listening on %v, tls", ln.Addr())
} else {
context.GetLogger(registry.app).Infof("listening on %v", ln.Addr())
}
return registry.server.Serve(ln)
}
func configureReporting(app *handlers.App) http.Handler {
var handler http.Handler = app
if app.Config.Reporting.Bugsnag.APIKey != "" {
bugsnagConfig := bugsnag.Configuration{
APIKey: app.Config.Reporting.Bugsnag.APIKey,
// TODO(brianbland): provide the registry version here
// AppVersion: "2.0",
}
if app.Config.Reporting.Bugsnag.ReleaseStage != "" {
bugsnagConfig.ReleaseStage = app.Config.Reporting.Bugsnag.ReleaseStage
}
if app.Config.Reporting.Bugsnag.Endpoint != "" {
bugsnagConfig.Endpoint = app.Config.Reporting.Bugsnag.Endpoint
}
bugsnag.Configure(bugsnagConfig)
handler = bugsnag.Handler(handler)
}
if app.Config.Reporting.NewRelic.LicenseKey != "" {
agent := gorelic.NewAgent()
agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey
if app.Config.Reporting.NewRelic.Name != "" {
agent.NewrelicName = app.Config.Reporting.NewRelic.Name
}
agent.CollectHTTPStat = true
agent.Verbose = app.Config.Reporting.NewRelic.Verbose
agent.Run()
handler = agent.WrapHTTPHandler(handler)
}
return handler
}
// configureLogging prepares the context with a logger using the
// configuration.
func configureLogging(ctx context.Context, config *configuration.Configuration) (context.Context, error) {
if config.Log.Level == "" && config.Log.Formatter == "" {
// If no config for logging is set, fallback to deprecated "Loglevel".
log.SetLevel(logLevel(config.Loglevel))
ctx = context.WithLogger(ctx, context.GetLogger(ctx))
return ctx, nil
}
log.SetLevel(logLevel(config.Log.Level))
formatter := config.Log.Formatter
if formatter == "" {
formatter = "text" // default formatter
}
switch formatter {
case "json":
log.SetFormatter(&log.JSONFormatter{
TimestampFormat: time.RFC3339Nano,
})
case "text":
log.SetFormatter(&log.TextFormatter{
TimestampFormat: time.RFC3339Nano,
})
case "logstash":
log.SetFormatter(&logstash.LogstashFormatter{
TimestampFormat: time.RFC3339Nano,
})
default:
// just let the library use default on empty string.
if config.Log.Formatter != "" {
return ctx, fmt.Errorf("unsupported logging formatter: %q", config.Log.Formatter)
}
}
if config.Log.Formatter != "" {
log.Debugf("using %q logging formatter", config.Log.Formatter)
}
if len(config.Log.Fields) > 0 {
// build up the static fields, if present.
var fields []interface{}
for k := range config.Log.Fields {
fields = append(fields, k)
}
ctx = context.WithValues(ctx, config.Log.Fields)
ctx = context.WithLogger(ctx, context.GetLogger(ctx, fields...))
}
return ctx, nil
}
func logLevel(level configuration.Loglevel) log.Level {
l, err := log.ParseLevel(string(level))
if err != nil {
l = log.InfoLevel
log.Warnf("error parsing level %q: %v, using %q ", level, err, l)
}
return l
}
// panicHandler add an HTTP handler to web app. The handler recover the happening
// panic. logrus.Panic transmits panic message to pre-config log hooks, which is
// defined in config.yml.
func panicHandler(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
log.Panic(fmt.Sprintf("%v", err))
}
}()
handler.ServeHTTP(w, r)
})
}
// alive simply wraps the handler with a route that always returns an http 200
// response when the path is matched. If the path is not matched, the request
// is passed to the provided handler. There is no guarantee of anything but
// that the server is up. Wrap with other handlers (such as health.Handler)
// for greater affect.
func alive(path string, handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == path {
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
return
}
handler.ServeHTTP(w, r)
})
}
func resolveConfiguration(args []string) (*configuration.Configuration, error) {
var configurationPath string
if len(args) > 0 {
configurationPath = args[0]
} else if os.Getenv("REGISTRY_CONFIGURATION_PATH") != "" {
configurationPath = os.Getenv("REGISTRY_CONFIGURATION_PATH")
}
if configurationPath == "" {
return nil, fmt.Errorf("configuration path unspecified")
}
fp, err := os.Open(configurationPath)
if err != nil {
return nil, err
}
defer fp.Close()
config, err := configuration.Parse(fp)
if err != nil {
return nil, fmt.Errorf("error parsing %s: %v", configurationPath, err)
}
return config, nil
}

84
docs/root.go
View file

@ -1,84 +0,0 @@
package registry
import (
"fmt"
"os"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/docker/distribution/version"
"github.com/docker/libtrust"
"github.com/spf13/cobra"
)
var showVersion bool
func init() {
RootCmd.AddCommand(ServeCmd)
RootCmd.AddCommand(GCCmd)
GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
RootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show the version and exit")
}
// RootCmd is the main command for the 'registry' binary.
var RootCmd = &cobra.Command{
Use: "registry",
Short: "`registry`",
Long: "`registry`",
Run: func(cmd *cobra.Command, args []string) {
if showVersion {
version.PrintVersion()
return
}
cmd.Usage()
},
}
var dryRun bool
// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
Use: "garbage-collect <config>",
Short: "`garbage-collect` deletes layers not referenced by any manifests",
Long: "`garbage-collect` deletes layers not referenced by any manifests",
Run: func(cmd *cobra.Command, args []string) {
config, err := resolveConfiguration(args)
if err != nil {
fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
cmd.Usage()
os.Exit(1)
}
driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
if err != nil {
fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
os.Exit(1)
}
ctx := context.Background()
ctx, err = configureLogging(ctx, config)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
os.Exit(1)
}
k, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
registry, err := storage.NewRegistry(ctx, driver, storage.Schema1SigningKey(k))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
os.Exit(1)
}
err = storage.MarkAndSweep(ctx, driver, registry, dryRun)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
os.Exit(1)
}
},
}

5489
docs/spec/api.md Normal file
View file

File diff suppressed because it is too large Load diff

1219
docs/spec/api.md.tmpl Normal file
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more