package storage import ( "fmt" "path" "strings" "github.com/opencontainers/go-digest" ) const ( storagePathVersion = "v2" // fixed storage layout version storagePathRoot = "/docker/registry/" // all driver paths have a prefix // TODO(stevvooe): Get rid of the "storagePathRoot". Initially, we though // storage path root would configurable for all drivers through this // package. In reality, we've found it simpler to do this on a per driver // basis. ) // pathFor maps paths based on "object names" and their ids. The "object // names" mapped by are internal to the storage system. // // The path layout in the storage backend is roughly as follows: // // /v2 // ├── blobs // │ └── // │ └── // └── repositories // └── // ├── _layers // │ └── // ├── _manifests // │ ├── revisions // │ │ └── // │ │ └── link // │ └── tags // │ └── // │ ├── current // │ │ └── link // │ └── index // │ └── // │ └── // │ └── link // └── _uploads // └── // ├── data // ├── hashstates // │ └── // │ └── // └── startedat // // The storage backend layout is broken up into a content-addressable blob // store and repositories. The content-addressable blob store holds most data // throughout the backend, keyed by algorithm and digests of the underlying // content. Access to the blob store is controlled through links from the // repository to blobstore. // // A repository is made up of layers, manifests and tags. The layers component // is just a directory of layers which are "linked" into a repository. A layer // can only be accessed through a qualified repository name if it is linked in // the repository. Uploads of layers are managed in the uploads directory, // which is key by upload id. When all data for an upload is received, the // data is moved into the blob store and the upload directory is deleted. // Abandoned uploads can be garbage collected by reading the startedat file // and removing uploads that have been active for longer than a certain time. // // The third component of the repository directory is the manifests store, // which is made up of a revision store and tag store. Manifests are stored in // the blob store and linked into the revision store. // While the registry can save all revisions of a manifest, no relationship is // implied as to the ordering of changes to a manifest. The tag store provides // support for name, tag lookups of manifests, using "current/link" under a // named tag directory. An index is maintained to support deletions of all // revisions of a given manifest tag. // // We cover the path formats implemented by this path mapper below. // // Manifests: // // manifestsPathSpec: /v2/repositories//_manifests // manifestRevisionsPathSpec: /v2/repositories//_manifests/revisions/ // manifestRevisionPathSpec: /v2/repositories//_manifests/revisions/// // manifestRevisionLinkPathSpec: /v2/repositories//_manifests/revisions///link // // Tags: // // manifestTagsPathSpec: /v2/repositories//_manifests/tags/ // manifestTagPathSpec: /v2/repositories//_manifests/tags// // manifestTagCurrentPathSpec: /v2/repositories//_manifests/tags//current/link // manifestTagIndexPathSpec: /v2/repositories//_manifests/tags//index/ // manifestTagIndexEntryPathSpec: /v2/repositories//_manifests/tags//index/// // manifestTagIndexEntryLinkPathSpec: /v2/repositories//_manifests/tags//index///link // // Blobs: // // layerLinkPathSpec: /v2/repositories//_layers///link // layersPathSpec: /v2/repositories//_layers // // Uploads: // // uploadDataPathSpec: /v2/repositories//_uploads//data // uploadStartedAtPathSpec: /v2/repositories//_uploads//startedat // uploadHashStatePathSpec: /v2/repositories//_uploads//hashstates// // // Blob Store: // // blobsPathSpec: /v2/blobs/ // blobPathSpec: /v2/blobs/// // blobDataPathSpec: /v2/blobs////data // // For more information on the semantic meaning of each path and their // contents, please see the path spec documentation. func pathFor(spec pathSpec) (string, error) { // Switch on the path object type and return the appropriate path. At // first glance, one may wonder why we don't use an interface to // accomplish this. By keep the formatting separate from the pathSpec, we // keep separate the path generation componentized. These specs could be // passed to a completely different mapper implementation and generate a // different set of paths. // // For example, imagine migrating from one backend to the other: one could // build a filesystem walker that converts a string path in one version, // to an intermediate path object, than can be consumed and mapped by the // other version. rootPrefix := []string{storagePathRoot, storagePathVersion} repoPrefix := append(rootPrefix, "repositories") switch v := spec.(type) { case manifestsPathSpec: return path.Join(append(repoPrefix, v.name, "_manifests")...), nil case manifestRevisionsPathSpec: return path.Join(append(repoPrefix, v.name, "_manifests", "revisions")...), nil case manifestRevisionPathSpec: components, err := digestPathComponents(v.revision, false) if err != nil { return "", err } return path.Join(append(append(repoPrefix, v.name, "_manifests", "revisions"), components...)...), nil case manifestRevisionLinkPathSpec: root, err := pathFor(manifestRevisionPathSpec(v)) if err != nil { return "", err } return path.Join(root, "link"), nil case manifestTagsPathSpec: return path.Join(append(repoPrefix, v.name, "_manifests", "tags")...), nil case manifestTagPathSpec: root, err := pathFor(manifestTagsPathSpec{ name: v.name, }) if err != nil { return "", err } return path.Join(root, v.tag), nil case manifestTagCurrentPathSpec: root, err := pathFor(manifestTagPathSpec(v)) if err != nil { return "", err } return path.Join(root, "current", "link"), nil case manifestTagIndexPathSpec: root, err := pathFor(manifestTagPathSpec(v)) if err != nil { return "", err } return path.Join(root, "index"), nil case manifestTagIndexEntryLinkPathSpec: root, err := pathFor(manifestTagIndexEntryPathSpec(v)) if err != nil { return "", err } return path.Join(root, "link"), nil case manifestTagIndexEntryPathSpec: root, err := pathFor(manifestTagIndexPathSpec{ name: v.name, tag: v.tag, }) if err != nil { return "", err } components, err := digestPathComponents(v.revision, false) if err != nil { return "", err } return path.Join(root, path.Join(components...)), nil case layerLinkPathSpec: components, err := digestPathComponents(v.digest, false) if err != nil { return "", err } // TODO(stevvooe): Right now, all blobs are linked under "_layers". If // we have future migrations, we may want to rename this to "_blobs". // A migration strategy would simply leave existing items in place and // write the new paths, commit a file then delete the old files. blobLinkPathComponents := append(repoPrefix, v.name, "_layers") return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil case layersPathSpec: return path.Join(append(repoPrefix, v.name, "_layers")...), nil case blobsPathSpec: blobsPathPrefix := append(rootPrefix, "blobs") return path.Join(blobsPathPrefix...), nil case blobPathSpec: components, err := digestPathComponents(v.digest, true) if err != nil { return "", err } blobPathPrefix := append(rootPrefix, "blobs") return path.Join(append(blobPathPrefix, components...)...), nil case blobDataPathSpec: components, err := digestPathComponents(v.digest, true) if err != nil { return "", err } components = append(components, "data") blobPathPrefix := append(rootPrefix, "blobs") return path.Join(append(blobPathPrefix, components...)...), nil case uploadDataPathSpec: return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "data")...), nil case uploadStartedAtPathSpec: return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "startedat")...), nil case uploadHashStatePathSpec: offset := fmt.Sprintf("%d", v.offset) if v.list { offset = "" // Limit to the prefix for listing offsets. } return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil case repositoriesRootPathSpec: return path.Join(repoPrefix...), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) } } // pathSpec is a type to mark structs as path specs. There is no // implementation because we'd like to keep the specs and the mappers // decoupled. type pathSpec interface { pathSpec() } // manifestPathSpec describes the directory path for a manifest. type manifestsPathSpec struct { name string } func (manifestsPathSpec) pathSpec() {} // manifestRevisionsPathSpec describes the directory path for // a manifest revision. type manifestRevisionsPathSpec struct { name string } func (manifestRevisionsPathSpec) pathSpec() {} // manifestRevisionPathSpec describes the components of the directory path for // a manifest revision. type manifestRevisionPathSpec struct { name string revision digest.Digest } func (manifestRevisionPathSpec) pathSpec() {} // manifestRevisionLinkPathSpec describes the path components required to look // up the data link for a revision of a manifest. If this file is not present, // the manifest blob is not available in the given repo. The contents of this // file should just be the digest. type manifestRevisionLinkPathSpec struct { name string revision digest.Digest } func (manifestRevisionLinkPathSpec) pathSpec() {} // manifestTagsPathSpec describes the path elements required to point to the // manifest tags directory. type manifestTagsPathSpec struct { name string } func (manifestTagsPathSpec) pathSpec() {} // manifestTagPathSpec describes the path elements required to point to the // manifest tag links files under a repository. These contain a blob id that // can be used to look up the data and signatures. type manifestTagPathSpec struct { name string tag string } func (manifestTagPathSpec) pathSpec() {} // manifestTagCurrentPathSpec describes the link to the current revision for a // given tag. type manifestTagCurrentPathSpec struct { name string tag string } func (manifestTagCurrentPathSpec) pathSpec() {} // manifestTagCurrentPathSpec describes the link to the index of revisions // with the given tag. type manifestTagIndexPathSpec struct { name string tag string } func (manifestTagIndexPathSpec) pathSpec() {} // manifestTagIndexEntryPathSpec contains the entries of the index by revision. type manifestTagIndexEntryPathSpec struct { name string tag string revision digest.Digest } func (manifestTagIndexEntryPathSpec) pathSpec() {} // manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a // manifest with given tag within the index. type manifestTagIndexEntryLinkPathSpec struct { name string tag string revision digest.Digest } func (manifestTagIndexEntryLinkPathSpec) pathSpec() {} // layersPathSpec contains the path for the layers inside a repo type layersPathSpec struct { name string } func (layersPathSpec) pathSpec() {} // layerLinkPathSpec specifies a path for a blob link, which is a file with a // blob id. The blob link will contain a content addressable blob id reference // into the blob store. The format of the contents is as follows: // // : // // The following example of the file contents is more illustrative: // // sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36 // // This indicates that there is a blob with the id/digest, calculated via // sha256 that can be fetched from the blob store. type layerLinkPathSpec struct { name string digest digest.Digest } func (layerLinkPathSpec) pathSpec() {} // blobAlgorithmReplacer does some very simple path sanitization for user // input. Paths should be "safe" before getting this far due to strict digest // requirements but we can add further path conversion here, if needed. var blobAlgorithmReplacer = strings.NewReplacer( "+", "/", ".", "/", ";", "/", ) // blobsPathSpec contains the path for the blobs directory type blobsPathSpec struct{} func (blobsPathSpec) pathSpec() {} // blobPathSpec contains the path for the registry global blob store. type blobPathSpec struct { digest digest.Digest } func (blobPathSpec) pathSpec() {} // blobDataPathSpec contains the path for the registry global blob store. For // now, this contains layer data, exclusively. type blobDataPathSpec struct { digest digest.Digest } func (blobDataPathSpec) pathSpec() {} // uploadDataPathSpec defines the path parameters of the data file for // uploads. type uploadDataPathSpec struct { name string id string } func (uploadDataPathSpec) pathSpec() {} // uploadStartedAtPathSpec defines the path parameters for the file that stores the // start time of an uploads. If it is missing, the upload is considered // unknown. Admittedly, the presence of this file is an ugly hack to make sure // we have a way to cleanup old or stalled uploads that doesn't rely on driver // FileInfo behavior. If we come up with a more clever way to do this, we // should remove this file immediately and rely on the startetAt field from // the client to enforce time out policies. type uploadStartedAtPathSpec struct { name string id string } func (uploadStartedAtPathSpec) pathSpec() {} // uploadHashStatePathSpec defines the path parameters for the file that stores // the hash function state of an upload at a specific byte offset. If `list` is // set, then the path mapper will generate a list prefix for all hash state // offsets for the upload identified by the name, id, and alg. type uploadHashStatePathSpec struct { name string id string alg digest.Algorithm offset int64 list bool } func (uploadHashStatePathSpec) pathSpec() {} // repositoriesRootPathSpec returns the root of repositories type repositoriesRootPathSpec struct{} func (repositoriesRootPathSpec) pathSpec() {} // digestPathComponents provides a consistent path breakdown for a given // digest. For a generic digest, it will be as follows: // // / // // If multilevel is true, the first two bytes of the digest will separate // groups of digest folder. It will be as follows: // // // func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) { if err := dgst.Validate(); err != nil { return nil, err } algorithm := blobAlgorithmReplacer.Replace(string(dgst.Algorithm())) hex := dgst.Encoded() prefix := []string{algorithm} var suffix []string if multilevel { suffix = append(suffix, hex[:2]) } suffix = append(suffix, hex) return append(prefix, suffix...), nil } // Reconstructs a digest from a path func digestFromPath(digestPath string) (digest.Digest, error) { digestPath = strings.TrimSuffix(digestPath, "/data") dir, hex := path.Split(digestPath) dir = path.Dir(dir) dir, next := path.Split(dir) // next is either the algorithm OR the first two characters in the hex string var algo string if next == hex[:2] { algo = path.Base(dir) } else { algo = next } dgst := digest.NewDigestFromEncoded(digest.Algorithm(algo), hex) return dgst, dgst.Validate() }