TrueCloudLab/restic
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #3524 from MichaelEischer/atomic-sftp

Browse source 
sftp: Implement atomic uploads
This commit is contained in:
Alexander Neumann 2022-03-21 11:08:22 +01:00 committed by GitHub
commit a350625554
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
changelog/unreleased/issue-3003 Normal file
View file

@ -0,0 +1,10 @@
Enhancement: Atomic uploads for SFTP
The SFTP backend did not upload files atomically. An interrupted upload could
leave an incomplete file behind which could prevent restic from accessing the
repository.
Uploads in the SFTP backend are now done atomically.
https://github.com/restic/restic/issues/3003
https://github.com/restic/restic/pull/3524

27
internal/backend/sftp/sftp.go
View file

@ -3,6 +3,8 @@ package sftp
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"hash"
"io"
@ -252,6 +254,17 @@ func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
// tempSuffix generates a random string suffix that should be sufficiently long
// to avoid accidential conflicts
func tempSuffix() string {
var nonce [16]byte
_, err := rand.Read(nonce[:])
if err != nil {
panic(err)
}
return hex.EncodeToString(nonce[:])
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
debug.Log("Save %v", h)
@ -264,10 +277,11 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}
filename := r.Filename(h)
tmpFilename := filename + "-restic-temp-" + tempSuffix()
dirname := r.Dirname(h)
// create new file
f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
f, err := r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
if r.IsNotExist(err) {
// error is caused by a missing directory, try to create it
@ -276,7 +290,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
debug.Log("error creating dir %v: %v", r.Dirname(h), mkdirErr)
} else {
// try again
f, err = r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
f, err = r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
}
}
@ -298,7 +312,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
rmErr := r.c.Remove(f.Name())
if rmErr != nil {
debug.Log("sftp: failed to remove broken file %v: %v",
filename, rmErr)
f.Name(), rmErr)
}
err = r.checkNoSpace(dirname, rd.Length(), err)
@ -318,7 +332,12 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}
err = f.Close()
return errors.Wrap(err, "Close")
if err != nil {
return errors.Wrap(err, "Close")
}
err = r.c.Rename(tmpFilename, filename)
return errors.Wrap(err, "Rename")
}
// checkNoSpace checks if err was likely caused by lack of available space