Option to configure proxy cache TTL

Currently when registry is run as proxy it tries to cleanup unused blobs
from its cache after 7 days which is hard-coded. This PR makes that
value configurable.

Co-authored-by: Shiming Zhang <wzshiming@foxmail.com>
Co-authored-by: Manish Tomar <manish.tomar@docker.com>
Signed-off-by: Shiming Zhang <wzshiming@foxmail.com>

configuration/configuration.go

@@ -655,6 +655,11 @@ type Proxy struct {
 
 	// Password of the hub user
 	Password string `yaml:"password"`
+
+	// TTL is the expiry time of the content and will be cleaned up when it expires
+	// if not set, defaults to 7 * 24 hours
+	// If set to zero, will never expire cache
+	TTL *time.Duration `yaml:"ttl,omitempty"`
 }
 
 // Parse parses an input configuration yaml document into a Configuration struct

docs/configuration.md

@@ -299,6 +299,7 @@ proxy:
   remoteurl: https://registry-1.docker.io
   username: [username]
   password: [password]
+  ttl: 168h
 compatibility:
   schema1:
     signingkeyfile: /etc/registry/key.json
@@ -1184,6 +1185,7 @@ proxy:
   remoteurl: https://registry-1.docker.io
   username: [username]
   password: [password]
+  ttl: 168h
 ```
 
 The `proxy` structure allows a registry to be configured as a pull-through cache
@@ -1197,6 +1199,7 @@ is unsupported.
 | `remoteurl`| yes     | The URL for the repository on Docker Hub.             |
 | `username` | no      | The username registered with Docker Hub which has access to the repository. |
 | `password` | no      | The password used to authenticate to Docker Hub using the username specified in `username`. |
+| `ttl`      | no      | Expire proxy cache configured in "storage" after this time. Cache 168h(7 days) by default, set to 0 to disable cache expiration, The suffix is one of `ns`, `us`, `ms`, `s`, `m`, or `h`. If you specify a value but omit the suffix, the value is interpreted as a number of nanoseconds. |
 
 
 To enable pulling private repositories (e.g. `batman/robin`) specify the

docs/recipes/mirror.md

@@ -91,6 +91,7 @@ proxy:
   remoteurl: https://registry-1.docker.io
   username: [username]
   password: [password]
+  ttl: 168h
 ```
 
 > **Warning**: If you specify a username and password, it's very important to

registry/proxy/proxyblobstore.go

@@ -6,18 +6,21 @@ import (
 	"net/http"
 	"strconv"
 	"sync"
+	"time"
+
+	"github.com/opencontainers/go-digest"
 
 	"github.com/distribution/distribution/v3"
 	dcontext "github.com/distribution/distribution/v3/context"
 	"github.com/distribution/distribution/v3/reference"
 	"github.com/distribution/distribution/v3/registry/proxy/scheduler"
-	"github.com/opencontainers/go-digest"
 )
 
 type proxyBlobStore struct {
 	localStore     distribution.BlobStore
 	remoteStore    distribution.BlobService
 	scheduler      *scheduler.TTLExpirationScheduler
+	ttl            *time.Duration
 	repositoryName reference.Named
 	authChallenger authChallenger
 }
@@ -146,7 +149,10 @@ func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter,
 			return
 		}
 
-		pbs.scheduler.AddBlob(blobRef, repositoryTTL)
+		if pbs.scheduler != nil && pbs.ttl != nil {
+			pbs.scheduler.AddBlob(blobRef, *pbs.ttl)
+		}
+
 	}(dgst)
 
 	_, err = pbs.copyContent(ctx, dgst, w)

registry/proxy/proxyblobstore_test.go

@@ -251,6 +251,18 @@ func TestProxyStoreGet(t *testing.T) {
 	}
 }
 
+func TestProxyStoreGetWithoutScheduler(t *testing.T) {
+	te := makeTestEnv(t, "foo/bar")
+	te.store.scheduler = nil
+
+	populate(t, te, 1, 10, 1)
+
+	_, err := te.store.Get(te.ctx, te.inRemote[0].Digest)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestProxyStoreStat(t *testing.T) {
 	te := makeTestEnv(t, "foo/bar")
 

registry/proxy/proxymanifeststore.go

@@ -4,22 +4,21 @@ import (
 	"context"
 	"time"
 
+	"github.com/opencontainers/go-digest"
+
 	"github.com/distribution/distribution/v3"
 	dcontext "github.com/distribution/distribution/v3/context"
 	"github.com/distribution/distribution/v3/reference"
 	"github.com/distribution/distribution/v3/registry/proxy/scheduler"
-	"github.com/opencontainers/go-digest"
 )
 
-// todo(richardscothern): from cache control header or config
-const repositoryTTL = 24 * 7 * time.Hour
-
 type proxyManifestStore struct {
 	ctx             context.Context
 	localManifests  distribution.ManifestService
 	remoteManifests distribution.ManifestService
 	repositoryName  reference.Named
 	scheduler       *scheduler.TTLExpirationScheduler
+	ttl             *time.Duration
 	authChallenger  authChallenger
 }
 
@@ -77,7 +76,10 @@ func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, optio
 			return nil, err
 		}
 
-		pms.scheduler.AddManifest(repoBlob, repositoryTTL)
+		if pms.scheduler != nil && pms.ttl != nil {
+			pms.scheduler.AddManifest(repoBlob, *pms.ttl)
+		}
+
 		// Ensure the manifest blob is cleaned up
 		// pms.scheduler.AddBlob(blobRef, repositoryTTL)
 

registry/proxy/proxymanifeststore_test.go

@@ -272,3 +272,24 @@ func TestProxyManifests(t *testing.T) {
 		t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
 	}
 }
+
+func TestProxyManifestsWithoutScheduler(t *testing.T) {
+	name := "foo/bar"
+	env := newManifestStoreTestEnv(t, name, "latest")
+	env.manifests.scheduler = nil
+
+	ctx := context.Background()
+	exists, err := env.manifests.Exists(ctx, env.manifestDigest)
+	if err != nil {
+		t.Fatalf("Error checking existence")
+	}
+	if !exists {
+		t.Errorf("Unexpected non-existent manifest")
+	}
+
+	// Get - should succeed without scheduler
+	_, err = env.manifests.Get(ctx, env.manifestDigest)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

registry/proxy/proxyregistry.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"sync"
+	"time"
 
 	"github.com/distribution/distribution/v3"
 	"github.com/distribution/distribution/v3/configuration"
@@ -20,10 +21,13 @@ import (
 	"github.com/distribution/distribution/v3/registry/storage/driver"
 )
 
+var repositoryTTL = 24 * 7 * time.Hour
+
 // proxyingRegistry fetches content from a remote registry and caches it locally
 type proxyingRegistry struct {
 	embedded       distribution.Namespace // provides local registry functionality
 	scheduler      *scheduler.TTLExpirationScheduler
+	ttl            *time.Duration
 	remoteURL      url.URL
 	authChallenger authChallenger
 }
@@ -36,61 +40,76 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
 	}
 
 	v := storage.NewVacuum(ctx, driver)
-	s := scheduler.New(ctx, driver, "/scheduler-state.json")
-	s.OnBlobExpire(func(ref reference.Reference) error {
-		var r reference.Canonical
-		var ok bool
-		if r, ok = ref.(reference.Canonical); !ok {
-			return fmt.Errorf("unexpected reference type : %T", ref)
-		}
 
-		repo, err := registry.Repository(ctx, r)
+	var s *scheduler.TTLExpirationScheduler
+	var ttl *time.Duration
+	if config.TTL == nil {
+		// Default TTL is 7 days
+		ttl = &repositoryTTL
+	} else if *config.TTL > 0 {
+		ttl = config.TTL
+	} else {
+		// TTL is disabled, never expire
+		ttl = nil
+	}
+
+	if ttl != nil {
+		s = scheduler.New(ctx, driver, "/scheduler-state.json")
+		s.OnBlobExpire(func(ref reference.Reference) error {
+			var r reference.Canonical
+			var ok bool
+			if r, ok = ref.(reference.Canonical); !ok {
+				return fmt.Errorf("unexpected reference type : %T", ref)
+			}
+
+			repo, err := registry.Repository(ctx, r)
+			if err != nil {
+				return err
+			}
+
+			blobs := repo.Blobs(ctx)
+
+			// Clear the repository reference and descriptor caches
+			err = blobs.Delete(ctx, r.Digest())
+			if err != nil {
+				return err
+			}
+
+			err = v.RemoveBlob(r.Digest().String())
+			if err != nil {
+				return err
+			}
+
+			return nil
+		})
+
+		s.OnManifestExpire(func(ref reference.Reference) error {
+			var r reference.Canonical
+			var ok bool
+			if r, ok = ref.(reference.Canonical); !ok {
+				return fmt.Errorf("unexpected reference type : %T", ref)
+			}
+
+			repo, err := registry.Repository(ctx, r)
+			if err != nil {
+				return err
+			}
+
+			manifests, err := repo.Manifests(ctx)
+			if err != nil {
+				return err
+			}
+			err = manifests.Delete(ctx, r.Digest())
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		err = s.Start()
 		if err != nil {
-			return err
+			return nil, err
 		}
-
-		blobs := repo.Blobs(ctx)
-
-		// Clear the repository reference and descriptor caches
-		err = blobs.Delete(ctx, r.Digest())
-		if err != nil {
-			return err
-		}
-
-		err = v.RemoveBlob(r.Digest().String())
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	s.OnManifestExpire(func(ref reference.Reference) error {
-		var r reference.Canonical
-		var ok bool
-		if r, ok = ref.(reference.Canonical); !ok {
-			return fmt.Errorf("unexpected reference type : %T", ref)
-		}
-
-		repo, err := registry.Repository(ctx, r)
-		if err != nil {
-			return err
-		}
-
-		manifests, err := repo.Manifests(ctx)
-		if err != nil {
-			return err
-		}
-		err = manifests.Delete(ctx, r.Digest())
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-
-	err = s.Start()
-	if err != nil {
-		return nil, err
 	}
 
 	cs, err := configureAuth(config.Username, config.Password, config.RemoteURL)
@@ -101,6 +120,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
 	return &proxyingRegistry{
 		embedded:  registry,
 		scheduler: s,
+		ttl:       ttl,
 		remoteURL: *remoteURL,
 		authChallenger: &remoteAuthChallenger{
 			remoteURL: *remoteURL,
@@ -161,6 +181,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
 			localStore:     localRepo.Blobs(ctx),
 			remoteStore:    remoteRepo.Blobs(ctx),
 			scheduler:      pr.scheduler,
+			ttl:            pr.ttl,
 			repositoryName: name,
 			authChallenger: pr.authChallenger,
 		},
@@ -170,6 +191,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
 			remoteManifests: remoteManifests,
 			ctx:             ctx,
 			scheduler:       pr.scheduler,
+			ttl:             pr.ttl,
 			authChallenger:  pr.authChallenger,
 		},
 		name: name,