forked from TrueCloudLab/restic
Fixed unexpected 'pack file cannot be listed' error
Fixes #1633 Signed-off-by: Igor Fedorenko <igor@ifedorenko.com>
This commit is contained in:
parent
9f0605766c
commit
19e7803ac6
3 changed files with 35 additions and 12 deletions
6
changelog/0.8.3/issue-1633
Normal file
6
changelog/0.8.3/issue-1633
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
Bugfix: Fixed unexpected 'pack file cannot be listed' error
|
||||||
|
|
||||||
|
Due to regression introduced in 0.8.2, `rebuild-index` and `prune` commands
|
||||||
|
failed to read pack files with size of 587, 588, 589 or 590 bytes.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/1633
|
|
@ -172,8 +172,11 @@ func (p *Packer) String() string {
|
||||||
|
|
||||||
const maxHeaderSize = 16 * 1024 * 1024
|
const maxHeaderSize = 16 * 1024 * 1024
|
||||||
|
|
||||||
|
// size of the header-length field at the end of the file
|
||||||
|
var headerLengthSize = binary.Size(uint32(0))
|
||||||
|
|
||||||
// we require at least one entry in the header, and one blob for a pack file
|
// we require at least one entry in the header, and one blob for a pack file
|
||||||
var minFileSize = entrySize + crypto.Extension
|
var minFileSize = entrySize + crypto.Extension + uint(headerLengthSize)
|
||||||
|
|
||||||
// number of header enries to download as part of header-length request
|
// number of header enries to download as part of header-length request
|
||||||
var eagerEntries = uint(15)
|
var eagerEntries = uint(15)
|
||||||
|
@ -197,10 +200,10 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
|
||||||
// only make second request if actual number of entries is greater than eagerEntries
|
// only make second request if actual number of entries is greater than eagerEntries
|
||||||
|
|
||||||
eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension)
|
eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension)
|
||||||
if int64(eagerHl) > size {
|
if int64(eagerHl)+int64(headerLengthSize) > size {
|
||||||
eagerHl = uint32(size) - uint32(binary.Size(uint32(0)))
|
eagerHl = uint32(size) - uint32(headerLengthSize)
|
||||||
}
|
}
|
||||||
eagerBuf := make([]byte, eagerHl+uint32(binary.Size(uint32(0))))
|
eagerBuf := make([]byte, eagerHl+uint32(headerLengthSize))
|
||||||
|
|
||||||
n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf)))
|
n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -228,7 +231,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
|
||||||
return nil, errors.Wrap(err, "readHeader")
|
return nil, errors.Wrap(err, "readHeader")
|
||||||
}
|
}
|
||||||
|
|
||||||
if int64(hl) > size-int64(binary.Size(hl)) {
|
if int64(hl) > size-int64(headerLengthSize) {
|
||||||
err := InvalidFileError{Message: "header is larger than file"}
|
err := InvalidFileError{Message: "header is larger than file"}
|
||||||
return nil, errors.Wrap(err, "readHeader")
|
return nil, errors.Wrap(err, "readHeader")
|
||||||
}
|
}
|
||||||
|
@ -248,7 +251,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
|
||||||
// need more header bytes
|
// need more header bytes
|
||||||
buf = make([]byte, hl)
|
buf = make([]byte, hl)
|
||||||
missingHl := hl - eagerHl
|
missingHl := hl - eagerHl
|
||||||
n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(binary.Size(hl)))
|
n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(headerLengthSize))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "ReadAt")
|
return nil, errors.Wrap(err, "ReadAt")
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,11 +22,11 @@ func (rd *countingReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
|
||||||
|
|
||||||
func TestReadHeaderEagerLoad(t *testing.T) {
|
func TestReadHeaderEagerLoad(t *testing.T) {
|
||||||
|
|
||||||
testReadHeader := func(entryCount uint, expectedReadInvocationCount int) {
|
testReadHeader := func(dataSize int, entryCount uint, expectedReadInvocationCount int) {
|
||||||
expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension)
|
expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension)
|
||||||
|
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
buf.Write(rtest.Random(0, 100)) // pack blobs data
|
buf.Write(rtest.Random(0, dataSize)) // pack blobs data
|
||||||
buf.Write(expectedHeader) // pack header
|
buf.Write(expectedHeader) // pack header
|
||||||
binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length
|
binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length
|
||||||
|
|
||||||
|
@ -39,8 +39,22 @@ func TestReadHeaderEagerLoad(t *testing.T) {
|
||||||
rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount)
|
rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
testReadHeader(1, 1)
|
// basic
|
||||||
testReadHeader(eagerEntries-1, 1)
|
testReadHeader(100, 1, 1)
|
||||||
testReadHeader(eagerEntries, 1)
|
|
||||||
testReadHeader(eagerEntries+1, 2)
|
// header entries == eager entries
|
||||||
|
testReadHeader(100, eagerEntries-1, 1)
|
||||||
|
testReadHeader(100, eagerEntries, 1)
|
||||||
|
testReadHeader(100, eagerEntries+1, 2)
|
||||||
|
|
||||||
|
// file size == eager header load size
|
||||||
|
eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension)
|
||||||
|
headerSize := int(1*entrySize) + crypto.Extension
|
||||||
|
dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0))
|
||||||
|
testReadHeader(dataSize-1, 1, 1)
|
||||||
|
testReadHeader(dataSize, 1, 1)
|
||||||
|
testReadHeader(dataSize+1, 1, 1)
|
||||||
|
testReadHeader(dataSize+2, 1, 1)
|
||||||
|
testReadHeader(dataSize+3, 1, 1)
|
||||||
|
testReadHeader(dataSize+4, 1, 1)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue