dkirillov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #1560 from RichardScothern/gc

Browse source 
Fix signature handling with GC.
This commit is contained in:
Richard Scothern 2016-03-30 17:46:04 -07:00
commit bf182a4d3f
3 changed files with 55 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
docs/garbagecollect.go
View file

@ -13,16 +13,17 @@ import (
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/docker/libtrust"
"github.com/spf13/cobra"
)
func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error {
// Construct a registry
registry, err := storage.NewRegistry(ctx, storageDriver)
if err != nil {
return fmt.Errorf("failed to construct registry: %v", err)
func emit(format string, a ...interface{}) {
if dryRun {
fmt.Printf(format+"\n", a...)
}
}
func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {
repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
if !ok {
@ -31,7 +32,9 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
// mark
markSet := make(map[digest.Digest]struct{})
err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
emit(repoName)
var err error
named, err := reference.ParseNamed(repoName)
if err != nil {
@ -54,6 +57,7 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
// Mark the manifest's blob
emit("%s: marking manifest %s ", repoName, dgst)
markSet[dgst] = struct{}{}
manifest, err := manifestService.Get(ctx, dgst)
@ -64,6 +68,7 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
descriptors := manifest.References()
for _, descriptor := range descriptors {
markSet[descriptor.Digest] = struct{}{}
emit("%s: marking blob %s", repoName, descriptor.Digest)
}
switch manifest.(type) {
@ -77,11 +82,13 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
}
for _, signatureDigest := range signatures {
emit("%s: marking signature %s", repoName, signatureDigest)
markSet[signatureDigest] = struct{}{}
}
break
case *schema2.DeserializedManifest:
config := manifest.(*schema2.DeserializedManifest).Config
emit("%s: marking configuration %s", repoName, config.Digest)
markSet[config.Digest] = struct{}{}
break
}
@ -110,9 +117,14 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
return fmt.Errorf("error enumerating blobs: %v", err)
}
emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
// Construct vacuum
vacuum := storage.NewVacuum(ctx, storageDriver)
for dgst := range deleteSet {
emit("blob eligible for deletion: %s", dgst)
if dryRun {
continue
}
err = vacuum.RemoveBlob(string(dgst))
if err != nil {
return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
@ -122,13 +134,18 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
return err
}
func init() {
GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything expect remove the blobs")
}
var dryRun bool
// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
Use: "garbage-collect <config>",
Short: "`garbage-collects` deletes layers not referenced by any manifests",
Long: "`garbage-collects` deletes layers not referenced by any manifests",
Short: "`garbage-collect` deletes layers not referenced by any manifests",
Long: "`garbage-collect` deletes layers not referenced by any manifests",
Run: func(cmd *cobra.Command, args []string) {
config, err := resolveConfiguration(args)
if err != nil {
fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
@ -149,7 +166,19 @@ var GCCmd = &cobra.Command{
os.Exit(1)
}
err = markAndSweep(ctx, driver)
k, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
os.Exit(1)
}
err = markAndSweep(ctx, driver, registry)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
os.Exit(1)

6
docs/garbagecollect_test.go
View file

@ -161,7 +161,7 @@ func TestNoDeletionNoEffect(t *testing.T) {
}
// Run GC
err = markAndSweep(context.Background(), inmemoryDriver)
err = markAndSweep(context.Background(), inmemoryDriver, registry)
if err != nil {
t.Fatalf("Failed mark and sweep: %v", err)
}
@ -193,7 +193,7 @@ func TestDeletionHasEffect(t *testing.T) {
manifests.Delete(ctx, image3.manifestDigest)
// Run GC
err = markAndSweep(context.Background(), inmemoryDriver)
err = markAndSweep(context.Background(), inmemoryDriver, registry)
if err != nil {
t.Fatalf("Failed mark and sweep: %v", err)
}
@ -327,7 +327,7 @@ func TestOrphanBlobDeleted(t *testing.T) {
uploadRandomSchema2Image(t, repo)
// Run GC
err = markAndSweep(context.Background(), inmemoryDriver)
err = markAndSweep(context.Background(), inmemoryDriver, registry)
if err != nil {
t.Fatalf("Failed mark and sweep: %v", err)
}

17
docs/storage/manifeststore.go
View file

@ -12,6 +12,7 @@ import (
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/registry/storage/driver"
)
// A ManifestHandler gets and puts manifests of a particular type.
@ -161,16 +162,22 @@ func (ms *manifestStore) GetSignatures(ctx context.Context, manifestDigest diges
return nil, err
}
signaturesPath = path.Join(signaturesPath, "sha256")
var digests []digest.Digest
alg := string(digest.SHA256)
signaturePaths, err := ms.blobStore.driver.List(ctx, path.Join(signaturesPath, alg))
signaturePaths, err := ms.blobStore.driver.List(ctx, signaturesPath)
if err != nil {
switch err.(type) {
case nil:
break
case driver.PathNotFoundError:
// Manifest may have been pushed with signature store disabled
return digests, nil
default:
return nil, err
}
var digests []digest.Digest
for _, sigPath := range signaturePaths {
sigdigest, err := digest.ParseDigest("sha256:" + path.Base(sigPath))
sigdigest, err := digest.ParseDigest(alg + ":" + path.Base(sigPath))
if err != nil {
// merely found not a digest
continue