forked from TrueCloudLab/restic
backends: add sanity check for the uploaded file size
Bugs in the error handling while uploading a file to the backend could cause incomplete files, e.g. https://github.com/golang/go/issues/42400 which could affect the local backend. Proactively add sanity checks which will treat an upload as failed if the reported upload size does not match the actual file size.
This commit is contained in:
parent
4526d5d197
commit
c73316a111
7 changed files with 40 additions and 4 deletions
|
@ -172,6 +172,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
|||
// read the data, in 100 MiB chunks
|
||||
buf := make([]byte, 100*1024*1024)
|
||||
var blocks []storage.Block
|
||||
uploadedBytes := 0
|
||||
|
||||
for {
|
||||
n, err := io.ReadFull(rd, buf)
|
||||
|
@ -188,6 +189,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
|||
}
|
||||
|
||||
buf = buf[:n]
|
||||
uploadedBytes += n
|
||||
|
||||
// upload it as a new "block", use the base64 hash for the ID
|
||||
h := restic.Hash(buf)
|
||||
|
@ -204,6 +206,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
|||
})
|
||||
}
|
||||
|
||||
// sanity check
|
||||
if uploadedBytes != int(rd.Length()) {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
|
||||
}
|
||||
|
||||
debug.Log("uploaded %d parts: %v", len(blocks), blocks)
|
||||
err = file.PutBlockList(blocks, nil)
|
||||
debug.Log("PutBlockList returned %v", err)
|
||||
|
|
|
@ -209,6 +209,10 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
|
|||
return errors.Wrap(err, "Copy")
|
||||
}
|
||||
|
||||
// sanity check
|
||||
if n != rd.Length() {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
|
||||
}
|
||||
return errors.Wrap(w.Close(), "Close")
|
||||
}
|
||||
|
||||
|
|
|
@ -245,6 +245,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
|
|||
}
|
||||
|
||||
debug.Log("%v -> %v bytes", objName, wbytes)
|
||||
// sanity check
|
||||
if wbytes != rd.Length() {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -118,11 +118,16 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
|
|||
}
|
||||
|
||||
// save data, then sync
|
||||
_, err = io.Copy(f, rd)
|
||||
wbytes, err := io.Copy(f, rd)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return errors.Wrap(err, "Write")
|
||||
}
|
||||
// sanity check
|
||||
if wbytes != rd.Length() {
|
||||
_ = f.Close()
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
|
||||
}
|
||||
|
||||
if err = f.Sync(); err != nil {
|
||||
pathErr, ok := err.(*os.PathError)
|
||||
|
|
|
@ -84,6 +84,11 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
|
|||
be.data[h] = buf
|
||||
debug.Log("saved %v bytes at %v", len(buf), h)
|
||||
|
||||
// sanity check
|
||||
if int64(len(buf)) != rd.Length() {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", len(buf), rd.Length())
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
|
|
|
@ -272,9 +272,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
|
|||
opts.ContentType = "application/octet-stream"
|
||||
|
||||
debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length())
|
||||
n, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
|
||||
info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
|
||||
|
||||
debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)
|
||||
debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err)
|
||||
|
||||
// sanity check
|
||||
if err != nil && info.Size != rd.Length() {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", info.Size, rd.Length())
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "client.PutObject")
|
||||
}
|
||||
|
|
|
@ -288,12 +288,18 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
|
|||
}
|
||||
|
||||
// save data, make sure to use the optimized sftp upload method
|
||||
_, err = f.ReadFrom(rd)
|
||||
wbytes, err := f.ReadFrom(rd)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return errors.Wrap(err, "Write")
|
||||
}
|
||||
|
||||
// sanity check
|
||||
if wbytes != rd.Length() {
|
||||
_ = f.Close()
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Close")
|
||||
|
|
Loading…
Reference in a new issue