From 8fe4ca4038e4cae82e8e3fc69cb424bf03ef4adc Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Thu, 27 Aug 2020 17:07:35 -0700 Subject: [PATCH] Option to configure proxy cache TTL Currently when registry is run as proxy it tries to cleanup unused blobs from its cache after 7 days which is hard-coded. This PR makes that value configurable. Co-authored-by: Shiming Zhang Co-authored-by: Manish Tomar Signed-off-by: Shiming Zhang --- configuration/configuration.go | 5 + docs/configuration.md | 3 + docs/recipes/mirror.md | 1 + registry/proxy/proxyblobstore.go | 10 +- registry/proxy/proxyblobstore_test.go | 12 +++ registry/proxy/proxymanifeststore.go | 12 ++- registry/proxy/proxymanifeststore_test.go | 21 ++++ registry/proxy/proxyregistry.go | 126 +++++++++++++--------- 8 files changed, 131 insertions(+), 59 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 1f773976b..6c7073b91 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -655,6 +655,11 @@ type Proxy struct { // Password of the hub user Password string `yaml:"password"` + + // TTL is the expiry time of the content and will be cleaned up when it expires + // if not set, defaults to 7 * 24 hours + // If set to zero, will never expire cache + TTL *time.Duration `yaml:"ttl,omitempty"` } // Parse parses an input configuration yaml document into a Configuration struct diff --git a/docs/configuration.md b/docs/configuration.md index 2c192e286..9fc5b7af7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -299,6 +299,7 @@ proxy: remoteurl: https://registry-1.docker.io username: [username] password: [password] + ttl: 168h compatibility: schema1: signingkeyfile: /etc/registry/key.json @@ -1184,6 +1185,7 @@ proxy: remoteurl: https://registry-1.docker.io username: [username] password: [password] + ttl: 168h ``` The `proxy` structure allows a registry to be configured as a pull-through cache @@ -1197,6 +1199,7 @@ is unsupported. | `remoteurl`| yes | The URL for the repository on Docker Hub. | | `username` | no | The username registered with Docker Hub which has access to the repository. | | `password` | no | The password used to authenticate to Docker Hub using the username specified in `username`. | +| `ttl` | no | Expire proxy cache configured in "storage" after this time. Cache 168h(7 days) by default, set to 0 to disable cache expiration, The suffix is one of `ns`, `us`, `ms`, `s`, `m`, or `h`. If you specify a value but omit the suffix, the value is interpreted as a number of nanoseconds. | To enable pulling private repositories (e.g. `batman/robin`) specify the diff --git a/docs/recipes/mirror.md b/docs/recipes/mirror.md index 2fd50ea55..50384f575 100644 --- a/docs/recipes/mirror.md +++ b/docs/recipes/mirror.md @@ -91,6 +91,7 @@ proxy: remoteurl: https://registry-1.docker.io username: [username] password: [password] + ttl: 168h ``` > **Warning**: If you specify a username and password, it's very important to diff --git a/registry/proxy/proxyblobstore.go b/registry/proxy/proxyblobstore.go index af34fa294..a03b72376 100644 --- a/registry/proxy/proxyblobstore.go +++ b/registry/proxy/proxyblobstore.go @@ -6,18 +6,21 @@ import ( "net/http" "strconv" "sync" + "time" + + "github.com/opencontainers/go-digest" "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/proxy/scheduler" - "github.com/opencontainers/go-digest" ) type proxyBlobStore struct { localStore distribution.BlobStore remoteStore distribution.BlobService scheduler *scheduler.TTLExpirationScheduler + ttl *time.Duration repositoryName reference.Named authChallenger authChallenger } @@ -146,7 +149,10 @@ func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, return } - pbs.scheduler.AddBlob(blobRef, repositoryTTL) + if pbs.scheduler != nil && pbs.ttl != nil { + pbs.scheduler.AddBlob(blobRef, *pbs.ttl) + } + }(dgst) _, err = pbs.copyContent(ctx, dgst, w) diff --git a/registry/proxy/proxyblobstore_test.go b/registry/proxy/proxyblobstore_test.go index b8ac992a6..1531ba38d 100644 --- a/registry/proxy/proxyblobstore_test.go +++ b/registry/proxy/proxyblobstore_test.go @@ -251,6 +251,18 @@ func TestProxyStoreGet(t *testing.T) { } } +func TestProxyStoreGetWithoutScheduler(t *testing.T) { + te := makeTestEnv(t, "foo/bar") + te.store.scheduler = nil + + populate(t, te, 1, 10, 1) + + _, err := te.store.Get(te.ctx, te.inRemote[0].Digest) + if err != nil { + t.Fatal(err) + } +} + func TestProxyStoreStat(t *testing.T) { te := makeTestEnv(t, "foo/bar") diff --git a/registry/proxy/proxymanifeststore.go b/registry/proxy/proxymanifeststore.go index fee5f08ec..5673e69bb 100644 --- a/registry/proxy/proxymanifeststore.go +++ b/registry/proxy/proxymanifeststore.go @@ -4,22 +4,21 @@ import ( "context" "time" + "github.com/opencontainers/go-digest" + "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/proxy/scheduler" - "github.com/opencontainers/go-digest" ) -// todo(richardscothern): from cache control header or config -const repositoryTTL = 24 * 7 * time.Hour - type proxyManifestStore struct { ctx context.Context localManifests distribution.ManifestService remoteManifests distribution.ManifestService repositoryName reference.Named scheduler *scheduler.TTLExpirationScheduler + ttl *time.Duration authChallenger authChallenger } @@ -77,7 +76,10 @@ func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, optio return nil, err } - pms.scheduler.AddManifest(repoBlob, repositoryTTL) + if pms.scheduler != nil && pms.ttl != nil { + pms.scheduler.AddManifest(repoBlob, *pms.ttl) + } + // Ensure the manifest blob is cleaned up // pms.scheduler.AddBlob(blobRef, repositoryTTL) diff --git a/registry/proxy/proxymanifeststore_test.go b/registry/proxy/proxymanifeststore_test.go index bd06a49b3..8db99b413 100644 --- a/registry/proxy/proxymanifeststore_test.go +++ b/registry/proxy/proxymanifeststore_test.go @@ -272,3 +272,24 @@ func TestProxyManifests(t *testing.T) { t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger) } } + +func TestProxyManifestsWithoutScheduler(t *testing.T) { + name := "foo/bar" + env := newManifestStoreTestEnv(t, name, "latest") + env.manifests.scheduler = nil + + ctx := context.Background() + exists, err := env.manifests.Exists(ctx, env.manifestDigest) + if err != nil { + t.Fatalf("Error checking existence") + } + if !exists { + t.Errorf("Unexpected non-existent manifest") + } + + // Get - should succeed without scheduler + _, err = env.manifests.Get(ctx, env.manifestDigest) + if err != nil { + t.Fatal(err) + } +} diff --git a/registry/proxy/proxyregistry.go b/registry/proxy/proxyregistry.go index 00f560daa..b67ba1a20 100644 --- a/registry/proxy/proxyregistry.go +++ b/registry/proxy/proxyregistry.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "sync" + "time" "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" @@ -20,10 +21,13 @@ import ( "github.com/distribution/distribution/v3/registry/storage/driver" ) +var repositoryTTL = 24 * 7 * time.Hour + // proxyingRegistry fetches content from a remote registry and caches it locally type proxyingRegistry struct { embedded distribution.Namespace // provides local registry functionality scheduler *scheduler.TTLExpirationScheduler + ttl *time.Duration remoteURL url.URL authChallenger authChallenger } @@ -36,61 +40,76 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name } v := storage.NewVacuum(ctx, driver) - s := scheduler.New(ctx, driver, "/scheduler-state.json") - s.OnBlobExpire(func(ref reference.Reference) error { - var r reference.Canonical - var ok bool - if r, ok = ref.(reference.Canonical); !ok { - return fmt.Errorf("unexpected reference type : %T", ref) - } - repo, err := registry.Repository(ctx, r) + var s *scheduler.TTLExpirationScheduler + var ttl *time.Duration + if config.TTL == nil { + // Default TTL is 7 days + ttl = &repositoryTTL + } else if *config.TTL > 0 { + ttl = config.TTL + } else { + // TTL is disabled, never expire + ttl = nil + } + + if ttl != nil { + s = scheduler.New(ctx, driver, "/scheduler-state.json") + s.OnBlobExpire(func(ref reference.Reference) error { + var r reference.Canonical + var ok bool + if r, ok = ref.(reference.Canonical); !ok { + return fmt.Errorf("unexpected reference type : %T", ref) + } + + repo, err := registry.Repository(ctx, r) + if err != nil { + return err + } + + blobs := repo.Blobs(ctx) + + // Clear the repository reference and descriptor caches + err = blobs.Delete(ctx, r.Digest()) + if err != nil { + return err + } + + err = v.RemoveBlob(r.Digest().String()) + if err != nil { + return err + } + + return nil + }) + + s.OnManifestExpire(func(ref reference.Reference) error { + var r reference.Canonical + var ok bool + if r, ok = ref.(reference.Canonical); !ok { + return fmt.Errorf("unexpected reference type : %T", ref) + } + + repo, err := registry.Repository(ctx, r) + if err != nil { + return err + } + + manifests, err := repo.Manifests(ctx) + if err != nil { + return err + } + err = manifests.Delete(ctx, r.Digest()) + if err != nil { + return err + } + return nil + }) + + err = s.Start() if err != nil { - return err + return nil, err } - - blobs := repo.Blobs(ctx) - - // Clear the repository reference and descriptor caches - err = blobs.Delete(ctx, r.Digest()) - if err != nil { - return err - } - - err = v.RemoveBlob(r.Digest().String()) - if err != nil { - return err - } - - return nil - }) - - s.OnManifestExpire(func(ref reference.Reference) error { - var r reference.Canonical - var ok bool - if r, ok = ref.(reference.Canonical); !ok { - return fmt.Errorf("unexpected reference type : %T", ref) - } - - repo, err := registry.Repository(ctx, r) - if err != nil { - return err - } - - manifests, err := repo.Manifests(ctx) - if err != nil { - return err - } - err = manifests.Delete(ctx, r.Digest()) - if err != nil { - return err - } - return nil - }) - - err = s.Start() - if err != nil { - return nil, err } cs, err := configureAuth(config.Username, config.Password, config.RemoteURL) @@ -101,6 +120,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name return &proxyingRegistry{ embedded: registry, scheduler: s, + ttl: ttl, remoteURL: *remoteURL, authChallenger: &remoteAuthChallenger{ remoteURL: *remoteURL, @@ -161,6 +181,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named localStore: localRepo.Blobs(ctx), remoteStore: remoteRepo.Blobs(ctx), scheduler: pr.scheduler, + ttl: pr.ttl, repositoryName: name, authChallenger: pr.authChallenger, }, @@ -170,6 +191,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named remoteManifests: remoteManifests, ctx: ctx, scheduler: pr.scheduler, + ttl: pr.ttl, authChallenger: pr.authChallenger, }, name: name,