From b8b5c8e8c9624d86961863dc389a1aa583d381b2 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 7 Jun 2017 21:59:01 +0200
Subject: [PATCH 1/6] s3: Rename struct to Backend

---
 src/restic/backend/s3/s3.go | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index 767c21dc0..5a1aefd11 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -21,8 +21,8 @@ import (
 
 const connLimit = 10
 
-// s3 is a backend which stores the data on an S3 endpoint.
-type s3 struct {
+// Backend stores data on an S3 endpoint.
+type Backend struct {
 	client       *minio.Client
 	sem          *backend.Semaphore
 	bucketname   string
@@ -32,8 +32,8 @@ type s3 struct {
 	backend.Layout
 }
 
-// make sure that *s3 implements backend.Backend
-var _ restic.Backend = &s3{}
+// make sure that *Backend implements backend.Backend
+var _ restic.Backend = &Backend{}
 
 const defaultLayout = "s3legacy"
 
@@ -52,7 +52,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, err
 	}
 
-	be := &s3{
+	be := &Backend{
 		client:       client,
 		sem:          sem,
 		bucketname:   cfg.Bucket,
@@ -87,13 +87,13 @@ func Open(cfg Config) (restic.Backend, error) {
 }
 
 // IsNotExist returns true if the error is caused by a not existing file.
-func (be *s3) IsNotExist(err error) bool {
+func (be *Backend) IsNotExist(err error) bool {
 	debug.Log("IsNotExist(%T, %#v)", err, err)
 	return os.IsNotExist(err)
 }
 
 // Join combines path components with slashes.
-func (be *s3) Join(p ...string) string {
+func (be *Backend) Join(p ...string) string {
 	return path.Join(p...)
 }
 
@@ -113,7 +113,7 @@ func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for
 func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)
 
 // ReadDir returns the entries for a directory.
-func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
+func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
 	debug.Log("ReadDir(%v)", dir)
 
 	// make sure dir ends with a slash
@@ -154,7 +154,7 @@ func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
 }
 
 // Location returns this backend's location (the bucket name).
-func (be *s3) Location() string {
+func (be *Backend) Location() string {
 	return be.bucketname
 }
 
@@ -203,7 +203,7 @@ func (wr preventCloser) Close() error {
 }
 
 // Save stores data in the backend at the handle.
-func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -259,7 +259,7 @@ func (wr wrapReader) Close() error {
 // Load returns a reader that yields the contents of the file at h at the
 // given offset. If length is nonzero, only a portion of the file is
 // returned. rd must be closed after use.
-func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
 	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
 	if err := h.Valid(); err != nil {
 		return nil, err
@@ -304,7 +304,7 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6
 }
 
 // Stat returns information about a blob.
-func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
 	debug.Log("%v", h)
 
 	objName := be.Filename(h)
@@ -334,7 +334,7 @@ func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, er
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) {
+func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
 	found := false
 	objName := be.Filename(h)
 	_, err := be.client.StatObject(be.bucketname, objName)
@@ -347,7 +347,7 @@ func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(ctx context.Context, h restic.Handle) error {
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
 	objName := be.Filename(h)
 	err := be.client.RemoveObject(be.bucketname, objName)
 	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
@@ -357,7 +357,7 @@ func (be *s3) Remove(ctx context.Context, h restic.Handle) error {
 // List returns a channel that yields all names of blobs of type t. A
 // goroutine is started for this. If the channel done is closed, sending
 // stops.
-func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string {
+func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
@@ -390,7 +390,7 @@ func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string {
 }
 
 // Remove keys for a specified backend type.
-func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error {
+func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
 	for key := range be.List(ctx, restic.DataFile) {
 		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
@@ -402,7 +402,7 @@ func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error {
 }
 
 // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
-func (be *s3) Delete(ctx context.Context) error {
+func (be *Backend) Delete(ctx context.Context) error {
 	alltypes := []restic.FileType{
 		restic.DataFile,
 		restic.KeyFile,
@@ -421,4 +421,4 @@ func (be *s3) Delete(ctx context.Context) error {
 }
 
 // Close does nothing
-func (be *s3) Close() error { return nil }
+func (be *Backend) Close() error { return nil }

From 7cf8f59987889839858e2ecaf418f839d934612c Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 7 Jun 2017 21:59:41 +0200
Subject: [PATCH 2/6] layout: Add String() and Name()

---
 src/restic/backend/layout.go          | 1 +
 src/restic/backend/layout_default.go  | 9 +++++++++
 src/restic/backend/layout_rest.go     | 9 +++++++++
 src/restic/backend/layout_s3legacy.go | 9 +++++++++
 4 files changed, 28 insertions(+)

diff --git a/src/restic/backend/layout.go b/src/restic/backend/layout.go
index 3d0953de8..b58f290be 100644
--- a/src/restic/backend/layout.go
+++ b/src/restic/backend/layout.go
@@ -17,6 +17,7 @@ type Layout interface {
 	Dirname(restic.Handle) string
 	Basedir(restic.FileType) string
 	Paths() []string
+	Name() string
 }
 
 // Filesystem is the abstraction of a file system used for a backend.
diff --git a/src/restic/backend/layout_default.go b/src/restic/backend/layout_default.go
index 665b85531..9b57657b5 100644
--- a/src/restic/backend/layout_default.go
+++ b/src/restic/backend/layout_default.go
@@ -19,6 +19,15 @@ var defaultLayoutPaths = map[restic.FileType]string{
 	restic.KeyFile:      "keys",
 }
 
+func (l *DefaultLayout) String() string {
+	return "<DefaultLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *DefaultLayout) Name() string {
+	return "default"
+}
+
 // Dirname returns the directory path for a given file type and name.
 func (l *DefaultLayout) Dirname(h restic.Handle) string {
 	p := defaultLayoutPaths[h.Type]
diff --git a/src/restic/backend/layout_rest.go b/src/restic/backend/layout_rest.go
index 2d01ece79..007be37c8 100644
--- a/src/restic/backend/layout_rest.go
+++ b/src/restic/backend/layout_rest.go
@@ -11,6 +11,15 @@ type RESTLayout struct {
 
 var restLayoutPaths = defaultLayoutPaths
 
+func (l *RESTLayout) String() string {
+	return "<RESTLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *RESTLayout) Name() string {
+	return "rest"
+}
+
 // Dirname returns the directory path for a given file type and name.
 func (l *RESTLayout) Dirname(h restic.Handle) string {
 	if h.Type == restic.ConfigFile {
diff --git a/src/restic/backend/layout_s3legacy.go b/src/restic/backend/layout_s3legacy.go
index 601d29bc5..d9d7ab212 100644
--- a/src/restic/backend/layout_s3legacy.go
+++ b/src/restic/backend/layout_s3legacy.go
@@ -18,6 +18,15 @@ var s3LayoutPaths = map[restic.FileType]string{
 	restic.KeyFile:      "key",
 }
 
+func (l *S3LegacyLayout) String() string {
+	return "<S3LegacyLayout>"
+}
+
+// Name returns the name for this layout.
+func (l *S3LegacyLayout) Name() string {
+	return "s3legacy"
+}
+
 // join calls Join with the first empty elements removed.
 func (l *S3LegacyLayout) join(url string, items ...string) string {
 	for len(items) > 0 && items[0] == "" {

From 4f9bf5312b6dc8269632e59f00bcc63666cef77f Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 7 Jun 2017 22:54:37 +0200
Subject: [PATCH 3/6] Add migrate

This commits adds a 'migrate' command and a migration to move s3
repositories from the 's3legacy' to the 'default' layout.
---
 src/cmds/restic/cmd_migrate.go     | 99 ++++++++++++++++++++++++++++++
 src/restic/backend/s3/s3.go        | 25 +++++++-
 src/restic/migrations/doc.go       |  2 +
 src/restic/migrations/interface.go | 21 +++++++
 src/restic/migrations/list.go      |  8 +++
 src/restic/migrations/s3_layout.go | 87 ++++++++++++++++++++++++++
 6 files changed, 241 insertions(+), 1 deletion(-)
 create mode 100644 src/cmds/restic/cmd_migrate.go
 create mode 100644 src/restic/migrations/doc.go
 create mode 100644 src/restic/migrations/interface.go
 create mode 100644 src/restic/migrations/list.go
 create mode 100644 src/restic/migrations/s3_layout.go

diff --git a/src/cmds/restic/cmd_migrate.go b/src/cmds/restic/cmd_migrate.go
new file mode 100644
index 000000000..6be04cc0b
--- /dev/null
+++ b/src/cmds/restic/cmd_migrate.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"restic"
+	"restic/migrations"
+
+	"github.com/spf13/cobra"
+)
+
+var cmdMigrate = &cobra.Command{
+	Use:   "migrate [name]",
+	Short: "apply migrations",
+	Long: `
+The "migrate" command applies migrations to a repository. When no migration
+name is explicitely given, a list of migrations that can be applied is printed.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runMigrate(migrateOptions, globalOptions, args)
+	},
+}
+
+// MigrateOptions bundles all options for the 'check' command.
+type MigrateOptions struct {
+}
+
+var migrateOptions MigrateOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdMigrate)
+}
+
+func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
+	ctx := gopts.ctx
+	Printf("available migrations:\n")
+	for _, m := range migrations.All {
+		ok, err := m.Check(ctx, repo)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			Printf("  %v\n", m.Name())
+		}
+	}
+
+	return nil
+}
+
+func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
+	ctx := gopts.ctx
+
+	var firsterr error
+	for _, name := range args {
+		for _, m := range migrations.All {
+			if m.Name() == name {
+				ok, err := m.Check(ctx, repo)
+				if err != nil {
+					return err
+				}
+
+				if !ok {
+					Warnf("migration %v cannot be applied: check failed\n")
+					continue
+				}
+
+				if err = m.Apply(ctx, repo); err != nil {
+					Warnf("migration %v failed: %v\n", m.Name(), err)
+					if firsterr == nil {
+						firsterr = err
+					}
+					continue
+				}
+
+				Printf("migration %v: success\n", m.Name())
+			}
+		}
+	}
+
+	return firsterr
+}
+
+func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	if len(args) == 0 {
+		return checkMigrations(opts, gopts, repo)
+	}
+
+	return applyMigrations(opts, gopts, repo, args)
+}
diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index 5a1aefd11..ef7908786 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -155,7 +155,12 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
 
 // Location returns this backend's location (the bucket name).
 func (be *Backend) Location() string {
-	return be.bucketname
+	return be.Join(be.bucketname, be.prefix)
+}
+
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.prefix
 }
 
 // getRemainingSize returns number of bytes remaining. If it is not possible to
@@ -422,3 +427,21 @@ func (be *Backend) Delete(ctx context.Context) error {
 
 // Close does nothing
 func (be *Backend) Close() error { return nil }
+
+// Rename moves a file based on the new layout l.
+func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
+	debug.Log("Rename %v to %v", h, l)
+	oldname := be.Filename(h)
+	newname := l.Filename(h)
+
+	debug.Log("  %v -> %v", oldname, newname)
+
+	coreClient := minio.Core{Client: be.client}
+	err := coreClient.CopyObject(be.bucketname, newname, path.Join(be.bucketname, oldname), minio.CopyConditions{})
+	if err != nil {
+		debug.Log("copy failed: %v", err)
+		return err
+	}
+
+	return be.client.RemoveObject(be.bucketname, oldname)
+}
diff --git a/src/restic/migrations/doc.go b/src/restic/migrations/doc.go
new file mode 100644
index 000000000..0c757fcf4
--- /dev/null
+++ b/src/restic/migrations/doc.go
@@ -0,0 +1,2 @@
+// Package migrations contains migrations that can be applied to a repository and/or backend.
+package migrations
diff --git a/src/restic/migrations/interface.go b/src/restic/migrations/interface.go
new file mode 100644
index 000000000..288ca273b
--- /dev/null
+++ b/src/restic/migrations/interface.go
@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"context"
+	"restic"
+)
+
+// Migration implements a data migration.
+type Migration interface {
+	// Check returns true if the migration can be applied to a repo.
+	Check(context.Context, restic.Repository) (bool, error)
+
+	// Apply runs the migration.
+	Apply(context.Context, restic.Repository) error
+
+	// Name returns a short name.
+	Name() string
+
+	// Descr returns a description what the migration does.
+	Desc() string
+}
diff --git a/src/restic/migrations/list.go b/src/restic/migrations/list.go
new file mode 100644
index 000000000..4442f343c
--- /dev/null
+++ b/src/restic/migrations/list.go
@@ -0,0 +1,8 @@
+package migrations
+
+// All contains all migrations.
+var All []Migration
+
+func register(m Migration) {
+	All = append(All, m)
+}
diff --git a/src/restic/migrations/s3_layout.go b/src/restic/migrations/s3_layout.go
new file mode 100644
index 000000000..75a83b885
--- /dev/null
+++ b/src/restic/migrations/s3_layout.go
@@ -0,0 +1,87 @@
+package migrations
+
+import (
+	"context"
+	"path"
+	"restic"
+	"restic/backend"
+	"restic/backend/s3"
+	"restic/debug"
+	"restic/errors"
+)
+
+func init() {
+	register(&S3Layout{})
+}
+
+// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the
+// "default" layout.
+type S3Layout struct{}
+
+// Check tests whether the migration can be applied.
+func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) {
+	be, ok := repo.Backend().(*s3.Backend)
+	if !ok {
+		debug.Log("backend is not s3")
+		return false, nil
+	}
+
+	if be.Layout.Name() != "s3legacy" {
+		debug.Log("layout is not s3legacy")
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
+	for name := range be.List(ctx, t) {
+		h := restic.Handle{Type: t, Name: name}
+		debug.Log("move %v", h)
+		if err := be.Rename(h, l); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Apply runs the migration.
+func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
+	be, ok := repo.Backend().(*s3.Backend)
+	if !ok {
+		debug.Log("backend is not s3")
+		return errors.New("backend is not s3")
+	}
+
+	newLayout := &backend.DefaultLayout{
+		Path: be.Path(),
+		Join: path.Join,
+	}
+
+	for _, t := range []restic.FileType{
+		restic.KeyFile,
+		restic.SnapshotFile,
+		restic.DataFile,
+		restic.LockFile,
+	} {
+		err := m.moveFiles(ctx, be, newLayout, t)
+		if err != nil {
+			return err
+		}
+	}
+
+	be.Layout = newLayout
+
+	return nil
+}
+
+// Name returns the name for this migration.
+func (m *S3Layout) Name() string {
+	return "s3_layout"
+}
+
+// Desc returns a short description what the migration does.
+func (m *S3Layout) Desc() string {
+	return "move files from 's3legacy' to the 'default' repository layout"
+}

From 04ded881f67df236eafe19a9ed4e0074e5f760e5 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 7 Jun 2017 23:08:20 +0200
Subject: [PATCH 4/6] s3: Change the default layout to "default"

---
 src/restic/backend/s3/s3.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index ef7908786..65c727613 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -35,7 +35,7 @@ type Backend struct {
 // make sure that *Backend implements backend.Backend
 var _ restic.Backend = &Backend{}
 
-const defaultLayout = "s3legacy"
+const defaultLayout = "default"
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
 // does not exist yet.

From ff3d2e42f472e6aedbb1773e069125d96bdabbba Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 8 Jun 2017 19:19:45 +0200
Subject: [PATCH 5/6] migrate: Be a bit more verbose

---
 src/cmds/restic/cmd_migrate.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cmds/restic/cmd_migrate.go b/src/cmds/restic/cmd_migrate.go
index 6be04cc0b..b585534a2 100644
--- a/src/cmds/restic/cmd_migrate.go
+++ b/src/cmds/restic/cmd_migrate.go
@@ -39,7 +39,7 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 		}
 
 		if ok {
-			Printf("  %v\n", m.Name())
+			Printf("  %v: %v\n", m.Name(), m.Desc())
 		}
 	}
 
@@ -59,10 +59,11 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 				}
 
 				if !ok {
-					Warnf("migration %v cannot be applied: check failed\n")
+					Warnf("migration %v cannot be applied: check failed\n", m.Name())
 					continue
 				}
 
+				Printf("applying migration %v...\n", m.Name())
 				if err = m.Apply(ctx, repo); err != nil {
 					Warnf("migration %v failed: %v\n", m.Name(), err)
 					if firsterr == nil {

From eb7ddd6e11d9971d77f0dac7430fbea6807cd95d Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 8 Jun 2017 19:21:52 +0200
Subject: [PATCH 6/6] Add entry to CHANGELOG

---
 CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92274a1be..801168bbf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,12 @@ Important Changes in 0.X.Y
    https://github.com/restic/restic/issues/989
    https://github.com/restic/restic/pull/993
 
+ * The default layout for the s3 backend is now `default` (instead of
+   `s3legacy`). Also, there's a new `migrate` command to convert an existing
+   repo, it can be run like this: `restic migrate s3_layout`
+   https://github.com/restic/restic/issues/965
+   https://github.com/restic/restic/pull/1004
+
 Important Changes in 0.6.1
 ==========================