Merge pull request #1077 from restic/create-subdirs

local/sftp: Auto-create subdirs of data/ on init/open

src/restic/backend/layout_default.go

@@ -1,6 +1,9 @@
 package backend
 
-import "restic"
+import (
+	"encoding/hex"
+	"restic"
+)
 
 // DefaultLayout implements the default layout for local and sftp backends, as
 // described in the Design document. The `data` directory has one level of
@@ -49,11 +52,18 @@ func (l *DefaultLayout) Filename(h restic.Handle) string {
 	return l.Join(l.Dirname(h), name)
 }
 
-// Paths returns all directory names
+// Paths returns all directory names needed for a repo.
 func (l *DefaultLayout) Paths() (dirs []string) {
 	for _, p := range defaultLayoutPaths {
 		dirs = append(dirs, l.Join(l.Path, p))
 	}
+
+	// also add subdirs
+	for i := 0; i < 256; i++ {
+		subdir := hex.EncodeToString([]byte{byte(i)})
+		dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.DataFile], subdir))
+	}
+
 	return dirs
 }
 

src/restic/backend/layout_test.go

@@ -111,6 +111,10 @@ func TestDefaultLayout(t *testing.T) {
 			filepath.Join(tempdir, "keys"),
 		}
 
+		for i := 0; i < 256; i++ {
+			want = append(want, filepath.Join(tempdir, "data", fmt.Sprintf("%02x", i)))
+		}
+
 		sort.Sort(sort.StringSlice(want))
 		sort.Sort(sort.StringSlice(dirs))
 

src/restic/backend/local/local.go

@@ -35,6 +35,14 @@ func Open(cfg Config) (*Local, error) {
 
 	be := &Local{Config: cfg, Layout: l}
 
+	// create paths for data and refs. MkdirAll does nothing if the directory already exists.
+	for _, d := range be.Paths() {
+		err := fs.MkdirAll(d, backend.Modes.Dir)
+		if err != nil {
+			return nil, errors.Wrap(err, "MkdirAll")
+		}
+	}
+
 	return be, nil
 }
 
@@ -89,26 +97,8 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err er
 
 	filename := b.Filename(h)
 
-	// create directories if necessary, ignore errors
-	if h.Type == restic.DataFile {
-		err = fs.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
-		if err != nil {
-			return errors.Wrap(err, "MkdirAll")
-		}
-	}
-
 	// create new file
 	f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
-	if os.IsNotExist(errors.Cause(err)) {
-		// create the locks dir, then try again
-		err = fs.MkdirAll(b.Dirname(h), backend.Modes.Dir)
-		if err != nil {
-			return errors.Wrap(err, "MkdirAll")
-		}
-
-		return b.Save(ctx, h, rd)
-	}
-
 	if err != nil {
 		return errors.Wrap(err, "OpenFile")
 	}

src/restic/backend/sftp/sftp.go

@@ -126,6 +126,15 @@ func Open(cfg Config) (*SFTP, error) {
 
 	debug.Log("layout: %v\n", sftp.Layout)
 
+	// create paths for data and refs. mkdirAll does nothing if the paths already exist.
+	for _, d := range sftp.Paths() {
+		err = sftp.mkdirAll(d, backend.Modes.Dir)
+		debug.Log("mkdirAll %v -> %v", d, err)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	sftp.Config = cfg
 	sftp.p = cfg.Path
 	return sftp, nil