distribution/registry/storage/garbagecollect.go

152 lines
4.4 KiB
Go
Raw Permalink Normal View History

package storage
import (
"context"
"fmt"
"github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/distribution/reference"
"github.com/opencontainers/go-digest"
)
func emit(format string, a ...interface{}) {
fmt.Printf(format+"\n", a...)
}
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
// GCOpts contains options for garbage collector
type GCOpts struct {
DryRun bool
RemoveUntagged bool
}
// ManifestDel contains manifest structure which will be deleted
type ManifestDel struct {
Name string
Digest digest.Digest
Tags []string
}
// MarkAndSweep performs a mark and sweep of registry data
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error {
repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
if !ok {
return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
}
// mark
markSet := make(map[digest.Digest]struct{})
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
manifestArr := make([]ManifestDel, 0)
err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
emit(repoName)
var err error
named, err := reference.WithName(repoName)
if err != nil {
return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
}
repository, err := registry.Repository(ctx, named)
if err != nil {
return fmt.Errorf("failed to construct repository: %v", err)
}
manifestService, err := repository.Manifests(ctx)
if err != nil {
return fmt.Errorf("failed to construct manifest service: %v", err)
}
manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
if !ok {
return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
}
err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
if opts.RemoveUntagged {
// fetch all tags where this manifest is the latest one
tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst})
if err != nil {
return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err)
}
if len(tags) == 0 {
emit("manifest eligible for deletion: %s", dgst)
// fetch all tags from repository
// all of these tags could contain manifest in history
// which means that we need check (and delete) those references when deleting manifest
allTags, err := repository.Tags(ctx).All(ctx)
if err != nil {
return fmt.Errorf("failed to retrieve tags %v", err)
}
manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags})
return nil
}
}
// Mark the manifest's blob
emit("%s: marking manifest %s ", repoName, dgst)
markSet[dgst] = struct{}{}
manifest, err := manifestService.Get(ctx, dgst)
if err != nil {
return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
}
descriptors := manifest.References()
for _, descriptor := range descriptors {
markSet[descriptor.Digest] = struct{}{}
emit("%s: marking blob %s", repoName, descriptor.Digest)
}
return nil
})
// In certain situations such as unfinished uploads, deleting all
// tags in S3 or removing the _manifests folder manually, this
// error may be of type PathNotFound.
//
// In these cases we can continue marking other manifests safely.
if _, ok := err.(driver.PathNotFoundError); ok {
return nil
}
return err
})
if err != nil {
return fmt.Errorf("failed to mark: %v", err)
}
// sweep
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
vacuum := NewVacuum(ctx, storageDriver)
if !opts.DryRun {
for _, obj := range manifestArr {
err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags)
if err != nil {
return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err)
}
}
}
blobService := registry.Blobs()
deleteSet := make(map[digest.Digest]struct{})
err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
// check if digest is in markSet. If not, delete it!
if _, ok := markSet[dgst]; !ok {
deleteSet[dgst] = struct{}{}
}
return nil
})
if err != nil {
return fmt.Errorf("error enumerating blobs: %v", err)
}
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr))
for dgst := range deleteSet {
emit("blob eligible for deletion: %s", dgst)
add possibility to clean untagged manifests add tests add possibility to clean untagged manifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add dry tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove underscores Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fixes Signed-off-by: Jesse Haka <haka.jesse@gmail.com> opts struct+use camelcase Signed-off-by: Jesse Haka <haka.jesse@gmail.com> doublecheck manifest in paths.go Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add gofmt Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix lint Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add log print Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move log to dryrun as well Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove counter Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove manifest tag references Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add tag to tests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> manifestsWithoutTags -> removeUntagged Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove RemoveManifestTagReferences and use removemanifests Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> remove pathfor Signed-off-by: Jesse Haka <haka.jesse@gmail.com> move removemanifest out of manifestenumerator, it does not work correctly if we delete stuff in it Signed-off-by: Jesse Haka <haka.jesse@gmail.com> add comment Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix context -> dcontext Signed-off-by: Jesse Haka <haka.jesse@gmail.com> fix gofmt
2017-06-06 08:02:47 +00:00
if opts.DryRun {
continue
}
err = vacuum.RemoveBlob(string(dgst))
if err != nil {
return fmt.Errorf("failed to delete blob %s: %v", dgst, err)
}
}
return err
}