backends: Add Save()

backend/handle.go

@@ -33,7 +33,7 @@ func (h Handle) Valid() error {
 	case Index:
 	case Config:
 	default:
-		return fmt.Errorf("invalid config %q", h.Type)
+		return fmt.Errorf("invalid Type %q", h.Type)
 	}
 
 	if h.Type == Config {

backend/local/backend_test.go

@@ -58,6 +58,20 @@ func TestLocalBackendWrite(t *testing.T) {
 	test.TestWrite(t)
 }
 
+func TestLocalBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestLocalBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
 func TestLocalBackendBackend(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/local/local.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
 )
 
 var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
@@ -231,12 +232,14 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) {
 		return err
 	}
 
-	f, err := os.Create(filename(b.p, h.Type, h.Name))
+	tmpfile, err := ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-")
 	if err != nil {
 		return err
 	}
 
-	n, err := f.Write(p)
+	debug.Log("local.Save", "save %v (%d bytes) to %v", h, len(p), tmpfile.Name())
+
+	n, err := tmpfile.Write(p)
 	if err != nil {
 		return err
 	}
@@ -245,11 +248,58 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) {
 		return errors.New("not all bytes writen")
 	}
 
-	if err = f.Sync(); err != nil {
+	if err = tmpfile.Sync(); err != nil {
 		return err
 	}
 
-	return f.Close()
+	err = tmpfile.Close()
+	if err != nil {
+		return err
+	}
+
+	f := filename(b.p, h.Type, h.Name)
+
+	// create directories if necessary, ignore errors
+	if h.Type == backend.Data {
+		os.MkdirAll(filepath.Dir(f), backend.Modes.Dir)
+	}
+
+	// test if new path already exists
+	if _, err := os.Stat(f); err == nil {
+		return fmt.Errorf("Rename(): file %v already exists", f)
+	}
+
+	err = os.Rename(tmpfile.Name(), f)
+	debug.Log("local.Save", "save %v: rename %v -> %v: %v",
+		h, filepath.Base(tmpfile.Name()), filepath.Base(f), err)
+
+	if err != nil {
+		return err
+	}
+
+	// set mode to read-only
+	fi, err := os.Stat(f)
+	if err != nil {
+		return err
+	}
+
+	err = setNewFileMode(f, fi)
+	if err != nil {
+		return err
+	}
+
+	// try to flush directory
+	d, err := os.Open(filepath.Dir(f))
+	if err != nil {
+		return err
+	}
+
+	err = d.Sync()
+	if err != nil {
+		return err
+	}
+
+	return d.Close()
 }
 
 // Stat returns information about a blob.

backend/mem/backend_test.go

@@ -58,6 +58,20 @@ func TestMemBackendWrite(t *testing.T) {
 	test.TestWrite(t)
 }
 
+func TestMemBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestMemBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
 func TestMemBackendBackend(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/mem/mem_backend.go

@@ -45,6 +45,10 @@ func New() *MemoryBackend {
 		return memLoad(be, h, p, off)
 	}
 
+	be.MockBackend.SaveFn = func(h backend.Handle, p []byte) error {
+		return memSave(be, h, p)
+	}
+
 	be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) {
 		return memStat(be, h)
 	}

backend/s3/backend_test.go

@@ -58,6 +58,20 @@ func TestS3BackendWrite(t *testing.T) {
 	test.TestWrite(t)
 }
 
+func TestS3BackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestS3BackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
 func TestS3BackendBackend(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/sftp/backend_test.go

@@ -58,6 +58,20 @@ func TestSftpBackendWrite(t *testing.T) {
 	test.TestWrite(t)
 }
 
+func TestSftpBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestSftpBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
 func TestSftpBackendBackend(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/sftp/sftp.go

@@ -15,6 +15,7 @@ import (
 	"github.com/juju/errors"
 	"github.com/pkg/sftp"
 	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
 )
 
 const (
@@ -379,12 +380,10 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
 		return err
 	}
 
-	f, err := r.c.Create(r.filename(h.Type, h.Name))
+	filename, tmpfile, err := r.tempFile()
-	if err != nil {
+	debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename)
-		return err
-	}
 
-	n, err := f.Write(p)
+	n, err := tmpfile.Write(p)
 	if err != nil {
 		return err
 	}
@@ -393,7 +392,19 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
 		return errors.New("not all bytes writen")
 	}
 
-	return f.Close()
+	err = tmpfile.Close()
+	if err != nil {
+		return err
+	}
+
+	err = r.renameFile(filename, h.Type, h.Name)
+	debug.Log("sftp.Save", "save %v: rename %v: %v",
+		h, filepath.Base(filename), err)
+	if err != nil {
+		return fmt.Errorf("sftp: renameFile: %v", err)
+	}
+
+	return nil
 }
 
 // Stat returns information about a blob.

backend/test/backend_test.go

@@ -58,6 +58,20 @@ func TestTestBackendWrite(t *testing.T) {
 	test.TestWrite(t)
 }
 
+func TestTestBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestTestBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
 func TestTestBackendBackend(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/test/tests.go

@@ -325,6 +325,93 @@ func TestWrite(t testing.TB) {
 	}
 }
 
+// TestSave tests saving data in the backend.
+func TestSave(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	length := rand.Intn(1<<23) + 2000
+	data := make([]byte, length)
+
+	for i := 0; i < 10; i++ {
+		_, err := io.ReadFull(crand.Reader, data)
+		OK(t, err)
+		id := backend.Hash(data)
+
+		h := backend.Handle{
+			Type: backend.Data,
+			Name: fmt.Sprintf("%s-%d", id, i),
+		}
+		err = b.Save(h, data)
+		OK(t, err)
+
+		buf, err := backend.LoadAll(b, h, nil)
+		OK(t, err)
+		if len(buf) != len(data) {
+			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
+		}
+
+		if !bytes.Equal(buf, data) {
+			t.Fatalf("data not equal")
+		}
+
+		fi, err := b.Stat(h)
+		OK(t, err)
+
+		if fi.Size != int64(len(data)) {
+			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
+		}
+
+		err = b.Remove(h.Type, h.Name)
+		if err != nil {
+			t.Fatalf("error removing item: %v", err)
+		}
+	}
+}
+
+var filenameTests = []struct {
+	name string
+	data string
+}{
+	{"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"},
+	{"foobar", "foobar"},
+	{
+		"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5",
+		"foobar content of data blob",
+	},
+}
+
+// TestSaveFilenames tests saving data with various file names in the backend.
+func TestSaveFilenames(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	for i, test := range filenameTests {
+		h := backend.Handle{Name: test.name, Type: backend.Data}
+		err := b.Save(h, []byte(test.data))
+		if err != nil {
+			t.Errorf("test %d failed: Save() returned %v", i, err)
+			continue
+		}
+
+		buf, err := backend.LoadAll(b, h, nil)
+		if err != nil {
+			t.Errorf("test %d failed: Load() returned %v", i, err)
+			continue
+		}
+
+		if !bytes.Equal(buf, []byte(test.data)) {
+			t.Errorf("test %d: returned wrong bytes", i)
+		}
+
+		err = b.Remove(h.Type, h.Name)
+		if err != nil {
+			t.Errorf("test %d failed: Remove() returned %v", i, err)
+			continue
+		}
+	}
+}
+
 var testStrings = []struct {
 	id   string
 	data string