Merge pull request #1560 from RichardScothern/gc

Fix signature handling with GC.

docs/garbagecollect.go

@@ -13,16 +13,17 @@ import (
 	"github.com/docker/distribution/registry/storage"
 	"github.com/docker/distribution/registry/storage/driver"
 	"github.com/docker/distribution/registry/storage/driver/factory"
-
+	"github.com/docker/libtrust"
 	"github.com/spf13/cobra"
 )
 
-func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error {
+func emit(format string, a ...interface{}) {
-	// Construct a registry
+	if dryRun {
-	registry, err := storage.NewRegistry(ctx, storageDriver)
+		fmt.Printf(format+"\n", a...)
-	if err != nil {
-		return fmt.Errorf("failed to construct registry: %v", err)
 	}
+}
+
+func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {
 
 	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
 	if !ok {
@@ -31,7 +32,9 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 
 	// mark
 	markSet := make(map[digest.Digest]struct{})
-	err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
+	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
+		emit(repoName)
+
 		var err error
 		named, err := reference.ParseNamed(repoName)
 		if err != nil {
@@ -54,6 +57,7 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 
 		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
 			// Mark the manifest's blob
+			emit("%s: marking manifest %s ", repoName, dgst)
 			markSet[dgst] = struct{}{}
 
 			manifest, err := manifestService.Get(ctx, dgst)
@@ -64,6 +68,7 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 			descriptors := manifest.References()
 			for _, descriptor := range descriptors {
 				markSet[descriptor.Digest] = struct{}{}
+				emit("%s: marking blob %s", repoName, descriptor.Digest)
 			}
 
 			switch manifest.(type) {
@@ -77,11 +82,13 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
 				}
 				for _, signatureDigest := range signatures {
+					emit("%s: marking signature %s", repoName, signatureDigest)
 					markSet[signatureDigest] = struct{}{}
 				}
 				break
 			case *schema2.DeserializedManifest:
 				config := manifest.(*schema2.DeserializedManifest).Config
+				emit("%s: marking configuration %s", repoName, config.Digest)
 				markSet[config.Digest] = struct{}{}
 				break
 			}
@@ -110,9 +117,14 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 		return fmt.Errorf("error enumerating blobs: %v", err)
 	}
 
+	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
 	// Construct vacuum
 	vacuum := storage.NewVacuum(ctx, storageDriver)
 	for dgst := range deleteSet {
+		emit("blob eligible for deletion: %s", dgst)
+		if dryRun {
+			continue
+		}
 		err = vacuum.RemoveBlob(string(dgst))
 		if err != nil {
 			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
@@ -122,13 +134,18 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error
 	return err
 }
 
+func init() {
+	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything expect remove the blobs")
+}
+
+var dryRun bool
+
 // GCCmd is the cobra command that corresponds to the garbage-collect subcommand
 var GCCmd = &cobra.Command{
 	Use:   "garbage-collect <config>",
-	Short: "`garbage-collects` deletes layers not referenced by any manifests",
+	Short: "`garbage-collect` deletes layers not referenced by any manifests",
-	Long:  "`garbage-collects` deletes layers not referenced by any manifests",
+	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
 	Run: func(cmd *cobra.Command, args []string) {
-
 		config, err := resolveConfiguration(args)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
@@ -149,7 +166,19 @@ var GCCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		err = markAndSweep(ctx, driver)
+		k, err := libtrust.GenerateECP256PrivateKey()
+		if err != nil {
+			fmt.Fprint(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
+			os.Exit(1)
+		}
+
+		err = markAndSweep(ctx, driver, registry)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
 			os.Exit(1)

docs/garbagecollect_test.go

@@ -161,7 +161,7 @@ func TestNoDeletionNoEffect(t *testing.T) {
 	}
 
 	// Run GC
-	err = markAndSweep(context.Background(), inmemoryDriver)
+	err = markAndSweep(context.Background(), inmemoryDriver, registry)
 	if err != nil {
 		t.Fatalf("Failed mark and sweep: %v", err)
 	}
@@ -193,7 +193,7 @@ func TestDeletionHasEffect(t *testing.T) {
 	manifests.Delete(ctx, image3.manifestDigest)
 
 	// Run GC
-	err = markAndSweep(context.Background(), inmemoryDriver)
+	err = markAndSweep(context.Background(), inmemoryDriver, registry)
 	if err != nil {
 		t.Fatalf("Failed mark and sweep: %v", err)
 	}
@@ -327,7 +327,7 @@ func TestOrphanBlobDeleted(t *testing.T) {
 	uploadRandomSchema2Image(t, repo)
 
 	// Run GC
-	err = markAndSweep(context.Background(), inmemoryDriver)
+	err = markAndSweep(context.Background(), inmemoryDriver, registry)
 	if err != nil {
 		t.Fatalf("Failed mark and sweep: %v", err)
 	}

docs/storage/manifeststore.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/registry/storage/driver"
 )
 
 // A ManifestHandler gets and puts manifests of a particular type.
@@ -161,16 +162,22 @@ func (ms *manifestStore) GetSignatures(ctx context.Context, manifestDigest diges
 		return nil, err
 	}
 
-	signaturesPath = path.Join(signaturesPath, "sha256")
+	var digests []digest.Digest
+	alg := string(digest.SHA256)
+	signaturePaths, err := ms.blobStore.driver.List(ctx, path.Join(signaturesPath, alg))
 
-	signaturePaths, err := ms.blobStore.driver.List(ctx, signaturesPath)
+	switch err.(type) {
-	if err != nil {
+	case nil:
+		break
+	case driver.PathNotFoundError:
+		// Manifest may have been pushed with signature store disabled
+		return digests, nil
+	default:
 		return nil, err
 	}
 
-	var digests []digest.Digest
 	for _, sigPath := range signaturePaths {
-		sigdigest, err := digest.ParseDigest("sha256:" + path.Base(sigPath))
+		sigdigest, err := digest.ParseDigest(alg + ":" + path.Base(sigPath))
 		if err != nil {
 			// merely found not a digest
 			continue