Merge pull request #1105 from restic/improve-sftp-open

sftp: Improve check for data subdirs

CHANGELOG.md

@@ -20,6 +20,7 @@ Small changes
    https://github.com/restic/rest-server/pull/11#issuecomment-309879710
    https://github.com/restic/restic/issues/1055
    https://github.com/restic/restic/pull/1077
+   https://github.com/restic/restic/pull/1105
 
  * When no S3 credentials are specified in the environment variables, restic
    now tries to load credentials from an IAM instance profile when the s3

src/restic/backend/sftp/sftp.go

@@ -126,13 +126,9 @@ func Open(cfg Config) (*SFTP, error) {
 
 	debug.Log("layout: %v\n", sftp.Layout)
 
-	// create paths for data and refs. mkdirAll does nothing if the paths already exist.
+	if err := sftp.checkDataSubdirs(); err != nil {
-	for _, d := range sftp.Paths() {
+		debug.Log("checkDataSubdirs returned %v", err)
-		err = sftp.mkdirAll(d, backend.Modes.Dir)
+		return nil, err
-		debug.Log("mkdirAll %v -> %v", d, err)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	sftp.Config = cfg
@@ -140,6 +136,46 @@ func Open(cfg Config) (*SFTP, error) {
 	return sftp, nil
 }
 
+func (r *SFTP) checkDataSubdirs() error {
+	datadir := r.Dirname(restic.Handle{Type: restic.DataFile})
+
+	// check if all paths for data/ exist
+	entries, err := r.c.ReadDir(datadir)
+	if err != nil {
+		return err
+	}
+
+	subdirs := make(map[string]struct{}, len(entries))
+	for _, entry := range entries {
+		subdirs[entry.Name()] = struct{}{}
+	}
+
+	for i := 0; i < 256; i++ {
+		subdir := fmt.Sprintf("%02x", i)
+		if _, ok := subdirs[subdir]; !ok {
+			debug.Log("subdir %v is missing, creating", subdir)
+			err := r.mkdirAll(path.Join(datadir, subdir), backend.Modes.Dir)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (r *SFTP) mkdirAllDataSubdirs() error {
+	for _, d := range r.Paths() {
+		err := r.mkdirAll(d, backend.Modes.Dir)
+		debug.Log("mkdirAll %v -> %v", d, err)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Join combines path components with slashes (according to the sftp spec).
 func (r *SFTP) Join(p ...string) string {
 	return path.Join(p...)
@@ -211,12 +247,8 @@ func Create(cfg Config) (*SFTP, error) {
 	}
 
 	// create paths for data and refs
-	for _, d := range sftp.Paths() {
+	if err = sftp.mkdirAllDataSubdirs(); err != nil {
-		err = sftp.mkdirAll(d, backend.Modes.Dir)
+		return nil, err
-		debug.Log("mkdirAll %v -> %v", d, err)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	err = sftp.Close()