Option to configure proxy cache TTL

Currently when registry is run as proxy it tries to cleanup unused blobs
from its cache after 7 days which is hard-coded. This PR makes that
value configurable.

Co-authored-by: Shiming Zhang <wzshiming@foxmail.com>
Co-authored-by: Manish Tomar <manish.tomar@docker.com>
Signed-off-by: Shiming Zhang <wzshiming@foxmail.com>
This commit is contained in:
Manish Tomar 2020-08-27 17:07:35 -07:00 committed by Shiming Zhang
parent 0c958010ac
commit 8fe4ca4038
8 changed files with 131 additions and 59 deletions

View file

@ -655,6 +655,11 @@ type Proxy struct {
// Password of the hub user // Password of the hub user
Password string `yaml:"password"` Password string `yaml:"password"`
// TTL is the expiry time of the content and will be cleaned up when it expires
// if not set, defaults to 7 * 24 hours
// If set to zero, will never expire cache
TTL *time.Duration `yaml:"ttl,omitempty"`
} }
// Parse parses an input configuration yaml document into a Configuration struct // Parse parses an input configuration yaml document into a Configuration struct

View file

@ -299,6 +299,7 @@ proxy:
remoteurl: https://registry-1.docker.io remoteurl: https://registry-1.docker.io
username: [username] username: [username]
password: [password] password: [password]
ttl: 168h
compatibility: compatibility:
schema1: schema1:
signingkeyfile: /etc/registry/key.json signingkeyfile: /etc/registry/key.json
@ -1184,6 +1185,7 @@ proxy:
remoteurl: https://registry-1.docker.io remoteurl: https://registry-1.docker.io
username: [username] username: [username]
password: [password] password: [password]
ttl: 168h
``` ```
The `proxy` structure allows a registry to be configured as a pull-through cache The `proxy` structure allows a registry to be configured as a pull-through cache
@ -1197,6 +1199,7 @@ is unsupported.
| `remoteurl`| yes | The URL for the repository on Docker Hub. | | `remoteurl`| yes | The URL for the repository on Docker Hub. |
| `username` | no | The username registered with Docker Hub which has access to the repository. | | `username` | no | The username registered with Docker Hub which has access to the repository. |
| `password` | no | The password used to authenticate to Docker Hub using the username specified in `username`. | | `password` | no | The password used to authenticate to Docker Hub using the username specified in `username`. |
| `ttl` | no | Expire proxy cache configured in "storage" after this time. Cache 168h(7 days) by default, set to 0 to disable cache expiration, The suffix is one of `ns`, `us`, `ms`, `s`, `m`, or `h`. If you specify a value but omit the suffix, the value is interpreted as a number of nanoseconds. |
To enable pulling private repositories (e.g. `batman/robin`) specify the To enable pulling private repositories (e.g. `batman/robin`) specify the

View file

@ -91,6 +91,7 @@ proxy:
remoteurl: https://registry-1.docker.io remoteurl: https://registry-1.docker.io
username: [username] username: [username]
password: [password] password: [password]
ttl: 168h
``` ```
> **Warning**: If you specify a username and password, it's very important to > **Warning**: If you specify a username and password, it's very important to

View file

@ -6,18 +6,21 @@ import (
"net/http" "net/http"
"strconv" "strconv"
"sync" "sync"
"time"
"github.com/opencontainers/go-digest"
"github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3"
dcontext "github.com/distribution/distribution/v3/context" dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/reference"
"github.com/distribution/distribution/v3/registry/proxy/scheduler" "github.com/distribution/distribution/v3/registry/proxy/scheduler"
"github.com/opencontainers/go-digest"
) )
type proxyBlobStore struct { type proxyBlobStore struct {
localStore distribution.BlobStore localStore distribution.BlobStore
remoteStore distribution.BlobService remoteStore distribution.BlobService
scheduler *scheduler.TTLExpirationScheduler scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
repositoryName reference.Named repositoryName reference.Named
authChallenger authChallenger authChallenger authChallenger
} }
@ -146,7 +149,10 @@ func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter,
return return
} }
pbs.scheduler.AddBlob(blobRef, repositoryTTL) if pbs.scheduler != nil && pbs.ttl != nil {
pbs.scheduler.AddBlob(blobRef, *pbs.ttl)
}
}(dgst) }(dgst)
_, err = pbs.copyContent(ctx, dgst, w) _, err = pbs.copyContent(ctx, dgst, w)

View file

@ -251,6 +251,18 @@ func TestProxyStoreGet(t *testing.T) {
} }
} }
func TestProxyStoreGetWithoutScheduler(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
te.store.scheduler = nil
populate(t, te, 1, 10, 1)
_, err := te.store.Get(te.ctx, te.inRemote[0].Digest)
if err != nil {
t.Fatal(err)
}
}
func TestProxyStoreStat(t *testing.T) { func TestProxyStoreStat(t *testing.T) {
te := makeTestEnv(t, "foo/bar") te := makeTestEnv(t, "foo/bar")

View file

@ -4,22 +4,21 @@ import (
"context" "context"
"time" "time"
"github.com/opencontainers/go-digest"
"github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3"
dcontext "github.com/distribution/distribution/v3/context" dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/reference"
"github.com/distribution/distribution/v3/registry/proxy/scheduler" "github.com/distribution/distribution/v3/registry/proxy/scheduler"
"github.com/opencontainers/go-digest"
) )
// todo(richardscothern): from cache control header or config
const repositoryTTL = 24 * 7 * time.Hour
type proxyManifestStore struct { type proxyManifestStore struct {
ctx context.Context ctx context.Context
localManifests distribution.ManifestService localManifests distribution.ManifestService
remoteManifests distribution.ManifestService remoteManifests distribution.ManifestService
repositoryName reference.Named repositoryName reference.Named
scheduler *scheduler.TTLExpirationScheduler scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
authChallenger authChallenger authChallenger authChallenger
} }
@ -77,7 +76,10 @@ func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, optio
return nil, err return nil, err
} }
pms.scheduler.AddManifest(repoBlob, repositoryTTL) if pms.scheduler != nil && pms.ttl != nil {
pms.scheduler.AddManifest(repoBlob, *pms.ttl)
}
// Ensure the manifest blob is cleaned up // Ensure the manifest blob is cleaned up
// pms.scheduler.AddBlob(blobRef, repositoryTTL) // pms.scheduler.AddBlob(blobRef, repositoryTTL)

View file

@ -272,3 +272,24 @@ func TestProxyManifests(t *testing.T) {
t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger) t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
} }
} }
func TestProxyManifestsWithoutScheduler(t *testing.T) {
name := "foo/bar"
env := newManifestStoreTestEnv(t, name, "latest")
env.manifests.scheduler = nil
ctx := context.Background()
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existence")
}
if !exists {
t.Errorf("Unexpected non-existent manifest")
}
// Get - should succeed without scheduler
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
}

View file

@ -6,6 +6,7 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"sync" "sync"
"time"
"github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/configuration"
@ -20,10 +21,13 @@ import (
"github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver"
) )
var repositoryTTL = 24 * 7 * time.Hour
// proxyingRegistry fetches content from a remote registry and caches it locally // proxyingRegistry fetches content from a remote registry and caches it locally
type proxyingRegistry struct { type proxyingRegistry struct {
embedded distribution.Namespace // provides local registry functionality embedded distribution.Namespace // provides local registry functionality
scheduler *scheduler.TTLExpirationScheduler scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
remoteURL url.URL remoteURL url.URL
authChallenger authChallenger authChallenger authChallenger
} }
@ -36,7 +40,21 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
} }
v := storage.NewVacuum(ctx, driver) v := storage.NewVacuum(ctx, driver)
s := scheduler.New(ctx, driver, "/scheduler-state.json")
var s *scheduler.TTLExpirationScheduler
var ttl *time.Duration
if config.TTL == nil {
// Default TTL is 7 days
ttl = &repositoryTTL
} else if *config.TTL > 0 {
ttl = config.TTL
} else {
// TTL is disabled, never expire
ttl = nil
}
if ttl != nil {
s = scheduler.New(ctx, driver, "/scheduler-state.json")
s.OnBlobExpire(func(ref reference.Reference) error { s.OnBlobExpire(func(ref reference.Reference) error {
var r reference.Canonical var r reference.Canonical
var ok bool var ok bool
@ -92,6 +110,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
if err != nil { if err != nil {
return nil, err return nil, err
} }
}
cs, err := configureAuth(config.Username, config.Password, config.RemoteURL) cs, err := configureAuth(config.Username, config.Password, config.RemoteURL)
if err != nil { if err != nil {
@ -101,6 +120,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
return &proxyingRegistry{ return &proxyingRegistry{
embedded: registry, embedded: registry,
scheduler: s, scheduler: s,
ttl: ttl,
remoteURL: *remoteURL, remoteURL: *remoteURL,
authChallenger: &remoteAuthChallenger{ authChallenger: &remoteAuthChallenger{
remoteURL: *remoteURL, remoteURL: *remoteURL,
@ -161,6 +181,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
localStore: localRepo.Blobs(ctx), localStore: localRepo.Blobs(ctx),
remoteStore: remoteRepo.Blobs(ctx), remoteStore: remoteRepo.Blobs(ctx),
scheduler: pr.scheduler, scheduler: pr.scheduler,
ttl: pr.ttl,
repositoryName: name, repositoryName: name,
authChallenger: pr.authChallenger, authChallenger: pr.authChallenger,
}, },
@ -170,6 +191,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
remoteManifests: remoteManifests, remoteManifests: remoteManifests,
ctx: ctx, ctx: ctx,
scheduler: pr.scheduler, scheduler: pr.scheduler,
ttl: pr.ttl,
authChallenger: pr.authChallenger, authChallenger: pr.authChallenger,
}, },
name: name, name: name,