alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

make offset and length uint32

Browse source
This commit is contained in:
Alexander Weiss 2020-06-12 08:17:14 +02:00
parent d92e2c5769
commit cf979e2b81
1 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
internal/repository/index.go
View file

@ -30,10 +30,10 @@ import (
// We have the following sizes: // We have the following sizes:
// key: 32 + 1 = 33 bytes // key: 32 + 1 = 33 bytes
// slice: 24 bytes (pointer, len and cap) // slice: 24 bytes (pointer, len and cap)
// indexEntry: 32 + 8 + 8 = 48 bytes // indexEntry: 32 + 4 + 4 = 40 bytes
// //
// To save N index entries, we therefore need: // To save N index entries, we therefore need:
// N * OF * (33 + 24) bytes + N * 48 bytes = N * 134 bytes // N * OF * (33 + 24) bytes + N * 40 bytes = N * 126 bytes
// Index holds a lookup table for id -> pack. // Index holds a lookup table for id -> pack.
type Index struct { type Index struct {
@ -49,8 +49,8 @@ type Index struct {
type indexEntry struct { type indexEntry struct {
packID restic.ID packID restic.ID
offset uint offset uint32
length uint length uint32
} }
// NewIndex returns a new index. // NewIndex returns a new index.
@ -61,11 +61,17 @@ func NewIndex() *Index {
} }
} }
const maxuint32 = 1<<32 - 1
func (idx *Index) store(blob restic.PackedBlob) { func (idx *Index) store(blob restic.PackedBlob) {
// assert that offset and length fit into uint32!
if blob.Offset > maxuint32 || blob.Length > maxuint32 {
panic("offset or length does not fit in uint32. You have packs > 4GB!")
}
newEntry := indexEntry{ newEntry := indexEntry{
packID: blob.PackID, packID: blob.PackID,
offset: blob.Offset, offset: uint32(blob.Offset),
length: blob.Length, length: uint32(blob.Length),
} }
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
idx.pack[h] = append(idx.pack[h], newEntry) idx.pack[h] = append(idx.pack[h], newEntry)
@ -146,8 +152,8 @@ func indexEntryToPackedBlob(h restic.BlobHandle, entry indexEntry) restic.Packed
Blob: restic.Blob{ Blob: restic.Blob{
ID: h.ID, ID: h.ID,
Type: h.Type, Type: h.Type,
Length: entry.length, Length: uint(entry.length),
Offset: entry.offset, Offset: uint(entry.offset),
}, },
PackID: entry.packID, PackID: entry.packID,
} }
@ -336,8 +342,8 @@ func (idx *Index) generatePackList() ([]*packJSON, error) {
p.Blobs = append(p.Blobs, blobJSON{ p.Blobs = append(p.Blobs, blobJSON{
ID: h.ID, ID: h.ID,
Type: h.Type, Type: h.Type,
Offset: blob.offset, Offset: uint(blob.offset),
Length: blob.length, Length: uint(blob.length),
}) })
} }
} }