Option to configure proxy cache TTL

Currently when registry is run as proxy it tries to cleanup unused blobs
from its cache after 7 days which is hard-coded. This PR makes that
value configurable.

Co-authored-by: Shiming Zhang <wzshiming@foxmail.com>
Co-authored-by: Manish Tomar <manish.tomar@docker.com>
Signed-off-by: Shiming Zhang <wzshiming@foxmail.com>
This commit is contained in:
Manish Tomar 2020-08-27 17:07:35 -07:00 committed by Shiming Zhang
parent 0c958010ac
commit 8fe4ca4038
8 changed files with 131 additions and 59 deletions

View file

@ -655,6 +655,11 @@ type Proxy struct {
// Password of the hub user
Password string `yaml:"password"`
// TTL is the expiry time of the content and will be cleaned up when it expires
// if not set, defaults to 7 * 24 hours
// If set to zero, will never expire cache
TTL *time.Duration `yaml:"ttl,omitempty"`
}
// Parse parses an input configuration yaml document into a Configuration struct

View file

@ -299,6 +299,7 @@ proxy:
remoteurl: https://registry-1.docker.io
username: [username]
password: [password]
ttl: 168h
compatibility:
schema1:
signingkeyfile: /etc/registry/key.json
@ -1184,6 +1185,7 @@ proxy:
remoteurl: https://registry-1.docker.io
username: [username]
password: [password]
ttl: 168h
```
The `proxy` structure allows a registry to be configured as a pull-through cache
@ -1197,6 +1199,7 @@ is unsupported.
| `remoteurl`| yes | The URL for the repository on Docker Hub. |
| `username` | no | The username registered with Docker Hub which has access to the repository. |
| `password` | no | The password used to authenticate to Docker Hub using the username specified in `username`. |
| `ttl` | no | Expire proxy cache configured in "storage" after this time. Cache 168h(7 days) by default, set to 0 to disable cache expiration, The suffix is one of `ns`, `us`, `ms`, `s`, `m`, or `h`. If you specify a value but omit the suffix, the value is interpreted as a number of nanoseconds. |
To enable pulling private repositories (e.g. `batman/robin`) specify the

View file

@ -91,6 +91,7 @@ proxy:
remoteurl: https://registry-1.docker.io
username: [username]
password: [password]
ttl: 168h
```
> **Warning**: If you specify a username and password, it's very important to

View file

@ -6,18 +6,21 @@ import (
"net/http"
"strconv"
"sync"
"time"
"github.com/opencontainers/go-digest"
"github.com/distribution/distribution/v3"
dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/reference"
"github.com/distribution/distribution/v3/registry/proxy/scheduler"
"github.com/opencontainers/go-digest"
)
type proxyBlobStore struct {
localStore distribution.BlobStore
remoteStore distribution.BlobService
scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
repositoryName reference.Named
authChallenger authChallenger
}
@ -146,7 +149,10 @@ func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter,
return
}
pbs.scheduler.AddBlob(blobRef, repositoryTTL)
if pbs.scheduler != nil && pbs.ttl != nil {
pbs.scheduler.AddBlob(blobRef, *pbs.ttl)
}
}(dgst)
_, err = pbs.copyContent(ctx, dgst, w)

View file

@ -251,6 +251,18 @@ func TestProxyStoreGet(t *testing.T) {
}
}
func TestProxyStoreGetWithoutScheduler(t *testing.T) {
te := makeTestEnv(t, "foo/bar")
te.store.scheduler = nil
populate(t, te, 1, 10, 1)
_, err := te.store.Get(te.ctx, te.inRemote[0].Digest)
if err != nil {
t.Fatal(err)
}
}
func TestProxyStoreStat(t *testing.T) {
te := makeTestEnv(t, "foo/bar")

View file

@ -4,22 +4,21 @@ import (
"context"
"time"
"github.com/opencontainers/go-digest"
"github.com/distribution/distribution/v3"
dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/reference"
"github.com/distribution/distribution/v3/registry/proxy/scheduler"
"github.com/opencontainers/go-digest"
)
// todo(richardscothern): from cache control header or config
const repositoryTTL = 24 * 7 * time.Hour
type proxyManifestStore struct {
ctx context.Context
localManifests distribution.ManifestService
remoteManifests distribution.ManifestService
repositoryName reference.Named
scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
authChallenger authChallenger
}
@ -77,7 +76,10 @@ func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, optio
return nil, err
}
pms.scheduler.AddManifest(repoBlob, repositoryTTL)
if pms.scheduler != nil && pms.ttl != nil {
pms.scheduler.AddManifest(repoBlob, *pms.ttl)
}
// Ensure the manifest blob is cleaned up
// pms.scheduler.AddBlob(blobRef, repositoryTTL)

View file

@ -272,3 +272,24 @@ func TestProxyManifests(t *testing.T) {
t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
}
}
func TestProxyManifestsWithoutScheduler(t *testing.T) {
name := "foo/bar"
env := newManifestStoreTestEnv(t, name, "latest")
env.manifests.scheduler = nil
ctx := context.Background()
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existence")
}
if !exists {
t.Errorf("Unexpected non-existent manifest")
}
// Get - should succeed without scheduler
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
}

View file

@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"sync"
"time"
"github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/configuration"
@ -20,10 +21,13 @@ import (
"github.com/distribution/distribution/v3/registry/storage/driver"
)
var repositoryTTL = 24 * 7 * time.Hour
// proxyingRegistry fetches content from a remote registry and caches it locally
type proxyingRegistry struct {
embedded distribution.Namespace // provides local registry functionality
scheduler *scheduler.TTLExpirationScheduler
ttl *time.Duration
remoteURL url.URL
authChallenger authChallenger
}
@ -36,61 +40,76 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
}
v := storage.NewVacuum(ctx, driver)
s := scheduler.New(ctx, driver, "/scheduler-state.json")
s.OnBlobExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
var s *scheduler.TTLExpirationScheduler
var ttl *time.Duration
if config.TTL == nil {
// Default TTL is 7 days
ttl = &repositoryTTL
} else if *config.TTL > 0 {
ttl = config.TTL
} else {
// TTL is disabled, never expire
ttl = nil
}
if ttl != nil {
s = scheduler.New(ctx, driver, "/scheduler-state.json")
s.OnBlobExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
if err != nil {
return err
}
blobs := repo.Blobs(ctx)
// Clear the repository reference and descriptor caches
err = blobs.Delete(ctx, r.Digest())
if err != nil {
return err
}
err = v.RemoveBlob(r.Digest().String())
if err != nil {
return err
}
return nil
})
s.OnManifestExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
if err != nil {
return err
}
manifests, err := repo.Manifests(ctx)
if err != nil {
return err
}
err = manifests.Delete(ctx, r.Digest())
if err != nil {
return err
}
return nil
})
err = s.Start()
if err != nil {
return err
return nil, err
}
blobs := repo.Blobs(ctx)
// Clear the repository reference and descriptor caches
err = blobs.Delete(ctx, r.Digest())
if err != nil {
return err
}
err = v.RemoveBlob(r.Digest().String())
if err != nil {
return err
}
return nil
})
s.OnManifestExpire(func(ref reference.Reference) error {
var r reference.Canonical
var ok bool
if r, ok = ref.(reference.Canonical); !ok {
return fmt.Errorf("unexpected reference type : %T", ref)
}
repo, err := registry.Repository(ctx, r)
if err != nil {
return err
}
manifests, err := repo.Manifests(ctx)
if err != nil {
return err
}
err = manifests.Delete(ctx, r.Digest())
if err != nil {
return err
}
return nil
})
err = s.Start()
if err != nil {
return nil, err
}
cs, err := configureAuth(config.Username, config.Password, config.RemoteURL)
@ -101,6 +120,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
return &proxyingRegistry{
embedded: registry,
scheduler: s,
ttl: ttl,
remoteURL: *remoteURL,
authChallenger: &remoteAuthChallenger{
remoteURL: *remoteURL,
@ -161,6 +181,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
localStore: localRepo.Blobs(ctx),
remoteStore: remoteRepo.Blobs(ctx),
scheduler: pr.scheduler,
ttl: pr.ttl,
repositoryName: name,
authChallenger: pr.authChallenger,
},
@ -170,6 +191,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
remoteManifests: remoteManifests,
ctx: ctx,
scheduler: pr.scheduler,
ttl: pr.ttl,
authChallenger: pr.authChallenger,
},
name: name,