Speed up restic init over slow SFTP links

pkg/sftp.Client.MkdirAll(d) does a Stat to determine if d exists and is
a directory, then a recursive call to create the parent, so the calls
for data/?? each take three round trips. Doing a Mkdir first should
eliminate two round trips for 255/256 data directories as well as all
but one of the top-level directories.

Also, we can do all of the calls concurrently. This may reintroduce some
of the Stat calls when multiple goroutines try to create the same
parent, but at the default number of connections, that should not be
much of a problem.
This commit is contained in:
greatroar 2022-07-30 12:24:04 +02:00
parent 23ebec717c
commit 2bdc40e612
3 changed files with 25 additions and 9 deletions

View file

@ -21,6 +21,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp"
"golang.org/x/sync/errgroup"
)
// SFTP is a backend in a directory accessed via SFTP.
@ -154,15 +155,29 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) {
return sftp, nil
}
func (r *SFTP) mkdirAllDataSubdirs() error {
func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error {
// Run multiple MkdirAll calls concurrently. These involve multiple
// round-trips and we do a lot of them, so this whole operation can be slow
// on high-latency links.
g, _ := errgroup.WithContext(ctx)
// Use errgroup's built-in semaphore, because r.sem is not initialized yet.
g.SetLimit(int(nconn))
for _, d := range r.Paths() {
err := r.c.MkdirAll(d)
if err != nil {
return err
}
d := d
g.Go(func() error {
// First try Mkdir. For most directories in Paths, this takes one
// round trip, not counting duplicate parent creations causes by
// concurrency. MkdirAll first does Stat, then recursive MkdirAll
// on the parent, so calls typically take three round trips.
if err := r.c.Mkdir(d); err == nil {
return nil
}
return r.c.MkdirAll(d)
})
}
return nil
return g.Wait()
}
// Join combines path components with slashes (according to the sftp spec).
@ -240,7 +255,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
}
// create paths for data and refs
if err = sftp.mkdirAllDataSubdirs(); err != nil {
if err = sftp.mkdirAllDataSubdirs(ctx, cfg.Connections); err != nil {
return nil, err
}