diff --git a/registry/storage/catalog.go b/registry/storage/catalog.go index 85c6cbe8f..d992770b1 100644 --- a/registry/storage/catalog.go +++ b/registry/storage/catalog.go @@ -27,7 +27,7 @@ func (reg *registry) Repositories(ctx context.Context, repos []string, last stri return 0, err } - err = reg.blobStore.driver.WalkWithStartAfterHint(ctx, root, last, func(fileInfo driver.FileInfo) error { + err = reg.blobStore.driver.Walk(ctx, root, func(fileInfo driver.FileInfo) error { err := handleRepository(fileInfo, root, last, func(repoPath string) error { foundRepos = append(foundRepos, repoPath) return nil @@ -43,7 +43,7 @@ func (reg *registry) Repositories(ctx context.Context, repos []string, last stri } return nil - }) + }, driver.WithStartAfterHint(last)) n = copy(repos, foundRepos) diff --git a/registry/storage/driver/azure/azure.go b/registry/storage/driver/azure/azure.go index 74b715a3d..c2cf92777 100644 --- a/registry/storage/driver/azure/azure.go +++ b/registry/storage/driver/azure/azure.go @@ -385,15 +385,8 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int // Walk traverses a filesystem defined within driver, starting // from the given path, calling f on each file and directory -func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { - return storagedriver.WalkFallback(ctx, d, path, f) -} - -// WalkWithStartAfterHint traverses a filesystem defined within driver, starting -// from the given path, calling f on each file and directory. The hint is ignored -// because it is not yet implemented. -func (d *driver) WalkWithStartAfterHint(ctx context.Context, path string, _ string, f storagedriver.WalkFn) error { - return d.Walk(ctx, path, f) +func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { + return storagedriver.WalkFallback(ctx, d, path, f, options...) } // directDescendants will find direct descendants (blobs or virtual containers) diff --git a/registry/storage/driver/base/base.go b/registry/storage/driver/base/base.go index 000072586..4ea3394a2 100644 --- a/registry/storage/driver/base/base.go +++ b/registry/storage/driver/base/base.go @@ -226,7 +226,7 @@ func (base *Base) URLFor(ctx context.Context, path string, options map[string]in } // Walk wraps Walk of underlying storage driver. -func (base *Base) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { +func (base *Base) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { ctx, done := dcontext.WithTrace(ctx) defer done("%s.Walk(%q)", base.Name(), path) @@ -234,5 +234,5 @@ func (base *Base) Walk(ctx context.Context, path string, f storagedriver.WalkFn) return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()} } - return base.setDriverName(base.StorageDriver.Walk(ctx, path, f)) + return base.setDriverName(base.StorageDriver.Walk(ctx, path, f, options...)) } diff --git a/registry/storage/driver/filesystem/driver.go b/registry/storage/driver/filesystem/driver.go index 154d78516..ade7b2838 100644 --- a/registry/storage/driver/filesystem/driver.go +++ b/registry/storage/driver/filesystem/driver.go @@ -290,15 +290,8 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int // Walk traverses a filesystem defined within driver, starting // from the given path, calling f on each file and directory -func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { - return storagedriver.WalkFallback(ctx, d, path, f) -} - -// WalkWithStartAfterHint traverses a filesystem defined within driver, starting -// from the given path, calling f on each file and directory. The hint is ignored -// because it is not yet implemented. -func (d *driver) WalkWithStartAfterHint(ctx context.Context, path string, _ string, f storagedriver.WalkFn) error { - return d.Walk(ctx, path, f) +func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { + return storagedriver.WalkFallback(ctx, d, path, f, options...) } // fullPath returns the absolute path of a key within the Driver's storage. diff --git a/registry/storage/driver/gcs/gcs.go b/registry/storage/driver/gcs/gcs.go index eb1d636b8..81e1dde6a 100644 --- a/registry/storage/driver/gcs/gcs.go +++ b/registry/storage/driver/gcs/gcs.go @@ -849,8 +849,8 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int // Walk traverses a filesystem defined within driver, starting // from the given path, calling f on each file -func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { - return storagedriver.WalkFallback(ctx, d, path, f) +func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { + return storagedriver.WalkFallback(ctx, d, path, f, options...) } func startSession(client *http.Client, bucket string, name string) (uri string, err error) { diff --git a/registry/storage/driver/inmemory/driver.go b/registry/storage/driver/inmemory/driver.go index a1b6c9f69..d9c30f382 100644 --- a/registry/storage/driver/inmemory/driver.go +++ b/registry/storage/driver/inmemory/driver.go @@ -244,15 +244,8 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int // Walk traverses a filesystem defined within driver, starting // from the given path, calling f on each file and directory -func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn) error { - return storagedriver.WalkFallback(ctx, d, path, f) -} - -// WalkWithStartAfterHint traverses a filesystem defined within driver, starting -// from the given path, calling f on each file and directory. The hint is ignored -// because it is not yet implemented. -func (d *driver) WalkWithStartAfterHint(ctx context.Context, path string, _ string, f storagedriver.WalkFn) error { - return d.Walk(ctx, path, f) +func (d *driver) Walk(ctx context.Context, path string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { + return storagedriver.WalkFallback(ctx, d, path, f, options...) } type writer struct { diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 65b83be6f..7fb7f2381 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -1038,15 +1038,12 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int // Walk traverses a filesystem defined within driver, starting // from the given path, calling f on each file -func (d *driver) Walk(ctx context.Context, from string, f storagedriver.WalkFn) error { - return d.WalkWithStartAfterHint(ctx, from, "", f) -} +func (d *driver) Walk(ctx context.Context, from string, f storagedriver.WalkFn, options ...func(*storagedriver.WalkOptions)) error { + walkOptions := &storagedriver.WalkOptions{} + for _, o := range options { + o(walkOptions) + } -// WalkWithStartAfterHint traverses a filesystem defined within driver, starting -// from the given path, calling f on each file and directory. The start after hint -// is passed to the ListObjectsV2 API so that AWS can pre-filter any paths that are -// lexographically before the last paged item. -func (d *driver) WalkWithStartAfterHint(ctx context.Context, from string, startAfterHint string, f storagedriver.WalkFn) error { path := from if !strings.HasSuffix(path, "/") { path = path + "/" @@ -1058,7 +1055,7 @@ func (d *driver) WalkWithStartAfterHint(ctx context.Context, from string, startA } var objectCount int64 - if err := d.doWalk(ctx, &objectCount, d.s3Path(path), prefix, startAfterHint, f); err != nil { + if err := d.doWalk(ctx, &objectCount, d.s3Path(path), prefix, walkOptions.StartAfterHint, f); err != nil { return err } diff --git a/registry/storage/driver/storagedriver.go b/registry/storage/driver/storagedriver.go index 542167ffd..b9ef6a14f 100644 --- a/registry/storage/driver/storagedriver.go +++ b/registry/storage/driver/storagedriver.go @@ -32,6 +32,19 @@ func (version Version) Minor() uint { // CurrentVersion is the current storage driver Version. const CurrentVersion Version = "0.1" +// WalkOptions provides options to the walk function that may adjust its behaviour +type WalkOptions struct { + // If StartAfterHint is set, the walk may start with the first item lexographically + // after the hint, but it is not guaranteed and drivers may start the walk from the path. + StartAfterHint string +} + +func WithStartAfterHint(startAfterHint string) func(*WalkOptions) { + return func(s *WalkOptions) { + s.StartAfterHint = startAfterHint + } +} + // StorageDriver defines methods that a Storage Driver must implement for a // filesystem-like key/value object storage. Storage Drivers are automatically // registered via an internal registration mechanism, and generally created @@ -90,12 +103,7 @@ type StorageDriver interface { // to a directory, the directory will not be entered and Walk // will continue the traversal. // If the returned error from the WalkFn is ErrFilledBuffer, processing stops. - Walk(ctx context.Context, path string, f WalkFn) error - - // WalkWithStartAfterHint traverses a filesystem defined within driver as in - // Walk. If startAfterHint is set, the walk may start with the first item lexographically - // after the hint, but it is not guaranteed and drivers may start the walk from the path. - WalkWithStartAfterHint(ctx context.Context, path string, startAfterHint string, f WalkFn) error + Walk(ctx context.Context, path string, f WalkFn, options ...func(*WalkOptions)) error } // FileWriter provides an abstraction for an opened writable file-like object in diff --git a/registry/storage/driver/walk.go b/registry/storage/driver/walk.go index f97cef897..63461a407 100644 --- a/registry/storage/driver/walk.go +++ b/registry/storage/driver/walk.go @@ -26,7 +26,8 @@ type WalkFn func(fileInfo FileInfo) error // If the returned error from the WalkFn is ErrSkipDir and fileInfo refers // to a directory, the directory will not be entered and Walk // will continue the traversal. If fileInfo refers to a normal file, processing stops -func WalkFallback(ctx context.Context, driver StorageDriver, from string, f WalkFn) error { +func WalkFallback(ctx context.Context, driver StorageDriver, from string, f WalkFn, options ...func(*WalkOptions)) error { + // No options are supported by the fallback, can be dropped _, err := doWalkFallback(ctx, driver, from, f) return err }