From 19e7803ac6c26b8ac3f4ca64b331ca2c89595bea Mon Sep 17 00:00:00 2001 From: Igor Fedorenko Date: Tue, 20 Feb 2018 21:28:37 -0500 Subject: [PATCH] Fixed unexpected 'pack file cannot be listed' error Fixes #1633 Signed-off-by: Igor Fedorenko --- changelog/0.8.3/issue-1633 | 6 ++++++ internal/pack/pack.go | 15 +++++++++------ internal/pack/pack_internal_test.go | 26 ++++++++++++++++++++------ 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 changelog/0.8.3/issue-1633 diff --git a/changelog/0.8.3/issue-1633 b/changelog/0.8.3/issue-1633 new file mode 100644 index 000000000..b39808a02 --- /dev/null +++ b/changelog/0.8.3/issue-1633 @@ -0,0 +1,6 @@ +Bugfix: Fixed unexpected 'pack file cannot be listed' error + +Due to regression introduced in 0.8.2, `rebuild-index` and `prune` commands +failed to read pack files with size of 587, 588, 589 or 590 bytes. + +https://github.com/restic/restic/issues/1633 diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 6583b224b..e700d284a 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -172,8 +172,11 @@ func (p *Packer) String() string { const maxHeaderSize = 16 * 1024 * 1024 +// size of the header-length field at the end of the file +var headerLengthSize = binary.Size(uint32(0)) + // we require at least one entry in the header, and one blob for a pack file -var minFileSize = entrySize + crypto.Extension +var minFileSize = entrySize + crypto.Extension + uint(headerLengthSize) // number of header enries to download as part of header-length request var eagerEntries = uint(15) @@ -197,10 +200,10 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { // only make second request if actual number of entries is greater than eagerEntries eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension) - if int64(eagerHl) > size { - eagerHl = uint32(size) - uint32(binary.Size(uint32(0))) + if int64(eagerHl)+int64(headerLengthSize) > size { + eagerHl = uint32(size) - uint32(headerLengthSize) } - eagerBuf := make([]byte, eagerHl+uint32(binary.Size(uint32(0)))) + eagerBuf := make([]byte, eagerHl+uint32(headerLengthSize)) n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf))) if err != nil { @@ -228,7 +231,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { return nil, errors.Wrap(err, "readHeader") } - if int64(hl) > size-int64(binary.Size(hl)) { + if int64(hl) > size-int64(headerLengthSize) { err := InvalidFileError{Message: "header is larger than file"} return nil, errors.Wrap(err, "readHeader") } @@ -248,7 +251,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { // need more header bytes buf = make([]byte, hl) missingHl := hl - eagerHl - n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(binary.Size(hl))) + n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(headerLengthSize)) if err != nil { return nil, errors.Wrap(err, "ReadAt") } diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index b83cd8f55..bd790609f 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -22,11 +22,11 @@ func (rd *countingReaderAt) ReadAt(p []byte, off int64) (n int, err error) { func TestReadHeaderEagerLoad(t *testing.T) { - testReadHeader := func(entryCount uint, expectedReadInvocationCount int) { + testReadHeader := func(dataSize int, entryCount uint, expectedReadInvocationCount int) { expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension) buf := &bytes.Buffer{} - buf.Write(rtest.Random(0, 100)) // pack blobs data + buf.Write(rtest.Random(0, dataSize)) // pack blobs data buf.Write(expectedHeader) // pack header binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length @@ -39,8 +39,22 @@ func TestReadHeaderEagerLoad(t *testing.T) { rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount) } - testReadHeader(1, 1) - testReadHeader(eagerEntries-1, 1) - testReadHeader(eagerEntries, 1) - testReadHeader(eagerEntries+1, 2) + // basic + testReadHeader(100, 1, 1) + + // header entries == eager entries + testReadHeader(100, eagerEntries-1, 1) + testReadHeader(100, eagerEntries, 1) + testReadHeader(100, eagerEntries+1, 2) + + // file size == eager header load size + eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension) + headerSize := int(1*entrySize) + crypto.Extension + dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0)) + testReadHeader(dataSize-1, 1, 1) + testReadHeader(dataSize, 1, 1) + testReadHeader(dataSize+1, 1, 1) + testReadHeader(dataSize+2, 1, 1) + testReadHeader(dataSize+3, 1, 1) + testReadHeader(dataSize+4, 1, 1) }