2015-05-09 21:52:03 +00:00
|
|
|
package repository
|
2015-04-26 15:10:31 +00:00
|
|
|
|
|
|
|
import (
|
2017-06-04 09:16:55 +00:00
|
|
|
"context"
|
2015-04-26 15:10:31 +00:00
|
|
|
"encoding/json"
|
|
|
|
"io"
|
|
|
|
"sync"
|
2015-10-12 21:59:17 +00:00
|
|
|
"time"
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2017-07-23 12:21:03 +00:00
|
|
|
"github.com/restic/restic/internal/errors"
|
2017-07-24 15:42:25 +00:00
|
|
|
"github.com/restic/restic/internal/restic"
|
2017-07-23 12:21:03 +00:00
|
|
|
|
|
|
|
"github.com/restic/restic/internal/debug"
|
2015-04-26 15:10:31 +00:00
|
|
|
)
|
|
|
|
|
2020-06-12 10:57:23 +00:00
|
|
|
// In large repositories, millions of blobs are stored in the repository
|
|
|
|
// and restic needs to store an index entry for each blob in memory for
|
|
|
|
// most operations.
|
|
|
|
// Hence the index data structure defined here is one of the main contributions
|
|
|
|
// to the total memory requirements of restic.
|
|
|
|
//
|
2020-06-23 20:13:25 +00:00
|
|
|
// We store the index entries in indexMaps. In these maps, entries take 56
|
|
|
|
// bytes each, plus 8/4 = 2 bytes of unused pointers on average, not counting
|
|
|
|
// malloc and header struct overhead and ignoring duplicates (those are only
|
|
|
|
// present in edge cases and are also removed by prune runs).
|
|
|
|
//
|
2020-06-12 06:25:49 +00:00
|
|
|
// In the index entries, we need to reference the packID. As one pack may
|
|
|
|
// contain many blobs the packIDs are saved in a separate array and only the index
|
|
|
|
// within this array is saved in the indexEntry
|
2020-06-12 10:57:23 +00:00
|
|
|
//
|
2020-06-23 20:13:25 +00:00
|
|
|
// We assume on average a minimum of 8 blobs per pack; BP=8.
|
2020-06-12 06:25:49 +00:00
|
|
|
// (Note that for large files there should be 3 blobs per pack as the average chunk
|
|
|
|
// size is 1.5 MB and the minimum pack size is 4 MB)
|
2020-06-12 10:57:23 +00:00
|
|
|
//
|
|
|
|
// We have the following sizes:
|
2020-06-23 20:13:25 +00:00
|
|
|
// indexEntry: 56 bytes (on amd64)
|
2020-06-12 06:25:49 +00:00
|
|
|
// each packID: 32 bytes
|
2020-06-12 10:57:23 +00:00
|
|
|
//
|
|
|
|
// To save N index entries, we therefore need:
|
2020-06-23 20:13:25 +00:00
|
|
|
// N * (56 + 2) bytes + N * 32 bytes / BP = N * 62 bytes,
|
|
|
|
// i.e., fewer than 64 bytes per blob in an index.
|
2020-06-12 10:57:23 +00:00
|
|
|
|
2020-06-12 06:25:49 +00:00
|
|
|
// Index holds lookup tables for id -> pack.
|
2015-04-26 15:10:31 +00:00
|
|
|
type Index struct {
|
2020-06-23 20:13:25 +00:00
|
|
|
m sync.Mutex
|
|
|
|
byType [restic.NumBlobTypes]indexMap
|
|
|
|
packs restic.IDs
|
|
|
|
treePacks restic.IDs
|
2020-06-13 10:43:03 +00:00
|
|
|
// only used by Store, StorePacks does not check for already saved packIDs
|
|
|
|
packIDToIndex map[restic.ID]int
|
2015-08-08 10:22:17 +00:00
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
final bool // set to true for all indexes read from the backend ("finalized")
|
|
|
|
ids restic.IDs // set to the IDs of the contained finalized indexes
|
2016-08-31 18:29:54 +00:00
|
|
|
supersedes restic.IDs
|
2015-10-12 21:59:17 +00:00
|
|
|
created time.Time
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewIndex returns a new index.
|
|
|
|
func NewIndex() *Index {
|
|
|
|
return &Index{
|
2020-06-13 10:43:03 +00:00
|
|
|
packIDToIndex: make(map[restic.ID]int),
|
|
|
|
created: time.Now(),
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-12 06:25:49 +00:00
|
|
|
// addToPacks saves the given pack ID and return the index.
|
|
|
|
// This procedere allows to use pack IDs which can be easily garbage collected after.
|
|
|
|
func (idx *Index) addToPacks(id restic.ID) int {
|
|
|
|
idx.packs = append(idx.packs, id)
|
|
|
|
return len(idx.packs) - 1
|
|
|
|
}
|
|
|
|
|
2020-06-12 06:17:14 +00:00
|
|
|
const maxuint32 = 1<<32 - 1
|
|
|
|
|
2020-06-12 06:25:49 +00:00
|
|
|
func (idx *Index) store(packIndex int, blob restic.Blob) {
|
2020-06-12 06:17:14 +00:00
|
|
|
// assert that offset and length fit into uint32!
|
|
|
|
if blob.Offset > maxuint32 || blob.Length > maxuint32 {
|
|
|
|
panic("offset or length does not fit in uint32. You have packs > 4GB!")
|
|
|
|
}
|
2020-06-23 20:13:25 +00:00
|
|
|
|
|
|
|
m := &idx.byType[blob.Type]
|
|
|
|
m.add(blob.ID, packIndex, uint32(blob.Offset), uint32(blob.Length))
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
// Final returns true iff the index is already written to the repository, it is
|
|
|
|
// finalized.
|
|
|
|
func (idx *Index) Final() bool {
|
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
return idx.final
|
|
|
|
}
|
|
|
|
|
2015-10-12 21:59:17 +00:00
|
|
|
const (
|
2020-05-24 07:19:42 +00:00
|
|
|
indexMaxBlobs = 50000
|
|
|
|
indexMaxAge = 10 * time.Minute
|
2015-10-12 21:59:17 +00:00
|
|
|
)
|
|
|
|
|
2015-10-25 14:05:22 +00:00
|
|
|
// IndexFull returns true iff the index is "full enough" to be saved as a preliminary index.
|
|
|
|
var IndexFull = func(idx *Index) bool {
|
2015-10-12 21:59:17 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("checking whether index %p is full", idx)
|
2015-10-12 21:59:17 +00:00
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
var blobs uint
|
|
|
|
for typ := range idx.byType {
|
|
|
|
blobs += idx.byType[typ].len()
|
|
|
|
}
|
2015-10-12 21:59:17 +00:00
|
|
|
age := time.Now().Sub(idx.created)
|
|
|
|
|
2020-05-24 07:19:42 +00:00
|
|
|
switch {
|
|
|
|
case age >= indexMaxAge:
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %p is old enough", idx, age)
|
2015-10-12 21:59:17 +00:00
|
|
|
return true
|
2020-05-24 07:19:42 +00:00
|
|
|
case blobs >= indexMaxBlobs:
|
|
|
|
debug.Log("index %p has %d blobs", idx, blobs)
|
2015-10-12 21:59:17 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-05-24 07:19:42 +00:00
|
|
|
debug.Log("index %p only has %d blobs and is too young (%v)", idx, blobs, age)
|
2015-10-12 21:59:17 +00:00
|
|
|
return false
|
2020-05-24 07:19:42 +00:00
|
|
|
|
2015-10-12 21:59:17 +00:00
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// Store remembers the id and pack in the index.
|
2016-08-31 20:39:36 +00:00
|
|
|
func (idx *Index) Store(blob restic.PackedBlob) {
|
2015-04-26 15:10:31 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
if idx.final {
|
|
|
|
panic("store new item in finalized index")
|
|
|
|
}
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("%v", blob)
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2020-06-13 10:43:03 +00:00
|
|
|
// get packIndex and save if new packID
|
|
|
|
packIndex, ok := idx.packIDToIndex[blob.PackID]
|
|
|
|
if !ok {
|
|
|
|
packIndex = idx.addToPacks(blob.PackID)
|
|
|
|
idx.packIDToIndex[blob.PackID] = packIndex
|
|
|
|
}
|
|
|
|
|
|
|
|
idx.store(packIndex, blob.Blob)
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// StorePack remembers the ids of all blobs of a given pack
|
|
|
|
// in the index
|
|
|
|
func (idx *Index) StorePack(id restic.ID, blobs []restic.Blob) {
|
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
if idx.final {
|
|
|
|
panic("store new item in finalized index")
|
|
|
|
}
|
|
|
|
|
|
|
|
debug.Log("%v", blobs)
|
2020-06-12 06:25:49 +00:00
|
|
|
packIndex := idx.addToPacks(id)
|
2020-06-06 20:20:44 +00:00
|
|
|
|
|
|
|
for _, blob := range blobs {
|
2020-06-12 06:25:49 +00:00
|
|
|
idx.store(packIndex, blob)
|
2020-06-06 20:20:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
func (idx *Index) toPackedBlob(e *indexEntry, typ restic.BlobType) restic.PackedBlob {
|
2020-02-08 19:51:50 +00:00
|
|
|
return restic.PackedBlob{
|
|
|
|
Blob: restic.Blob{
|
2020-06-23 20:13:25 +00:00
|
|
|
ID: e.id,
|
|
|
|
Type: typ,
|
|
|
|
Length: uint(e.length),
|
|
|
|
Offset: uint(e.offset),
|
2020-02-08 19:51:50 +00:00
|
|
|
},
|
2020-06-23 20:13:25 +00:00
|
|
|
PackID: idx.packs[e.packIndex],
|
2020-02-08 19:51:50 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-14 11:26:10 +00:00
|
|
|
// Lookup queries the index for the blob ID and returns all entries including
|
|
|
|
// duplicates. Adds found entries to blobs and returns the result.
|
|
|
|
func (idx *Index) Lookup(id restic.ID, tpe restic.BlobType, blobs []restic.PackedBlob) []restic.PackedBlob {
|
2015-04-26 15:10:31 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
idx.byType[tpe].foreachWithID(id, func(e *indexEntry) {
|
|
|
|
blobs = append(blobs, idx.toPackedBlob(e, tpe))
|
|
|
|
})
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2020-06-14 11:26:10 +00:00
|
|
|
return blobs
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2015-11-01 21:32:28 +00:00
|
|
|
// ListPack returns a list of blobs contained in a pack.
|
2016-08-31 20:39:36 +00:00
|
|
|
func (idx *Index) ListPack(id restic.ID) (list []restic.PackedBlob) {
|
2015-11-01 21:32:28 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
for typ := range idx.byType {
|
|
|
|
m := &idx.byType[typ]
|
|
|
|
m.foreach(func(e *indexEntry) bool {
|
|
|
|
if idx.packs[e.packIndex] == id {
|
|
|
|
list = append(list, idx.toPackedBlob(e, restic.BlobType(typ)))
|
2016-08-03 18:03:52 +00:00
|
|
|
}
|
2020-06-23 20:13:25 +00:00
|
|
|
return true
|
|
|
|
})
|
2015-11-01 21:32:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return list
|
|
|
|
}
|
|
|
|
|
2015-04-26 15:10:31 +00:00
|
|
|
// Has returns true iff the id is listed in the index.
|
2016-08-31 18:58:57 +00:00
|
|
|
func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
|
2018-01-08 18:38:21 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
return idx.byType[tpe].get(id) != nil
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2017-01-13 11:57:05 +00:00
|
|
|
// LookupSize returns the length of the plaintext content of the blob with the
|
|
|
|
// given id.
|
2018-01-12 06:20:12 +00:00
|
|
|
func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, found bool) {
|
2020-06-23 20:13:25 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
2016-08-03 20:38:05 +00:00
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
e := idx.byType[tpe].get(id)
|
|
|
|
if e == nil {
|
|
|
|
return 0, false
|
|
|
|
}
|
|
|
|
return uint(restic.PlaintextLength(int(e.length))), true
|
2015-07-19 13:16:16 +00:00
|
|
|
}
|
|
|
|
|
2015-08-08 10:22:17 +00:00
|
|
|
// Supersedes returns the list of indexes this index supersedes, if any.
|
2016-08-31 18:29:54 +00:00
|
|
|
func (idx *Index) Supersedes() restic.IDs {
|
2015-08-08 10:22:17 +00:00
|
|
|
return idx.supersedes
|
|
|
|
}
|
|
|
|
|
2015-10-25 16:06:56 +00:00
|
|
|
// AddToSupersedes adds the ids to the list of indexes superseded by this
|
|
|
|
// index. If the index has already been finalized, an error is returned.
|
2016-08-31 18:29:54 +00:00
|
|
|
func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
|
2015-10-25 16:06:56 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
if idx.final {
|
|
|
|
return errors.New("index already finalized")
|
|
|
|
}
|
|
|
|
|
|
|
|
idx.supersedes = append(idx.supersedes, ids...)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-06-18 12:45:02 +00:00
|
|
|
// Each returns a channel that yields all blobs known to the index. When the
|
|
|
|
// context is cancelled, the background goroutine terminates. This blocks any
|
|
|
|
// modification of the index.
|
|
|
|
func (idx *Index) Each(ctx context.Context) <-chan restic.PackedBlob {
|
2015-04-26 15:10:31 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
|
2016-08-31 20:39:36 +00:00
|
|
|
ch := make(chan restic.PackedBlob)
|
2015-04-26 15:10:31 +00:00
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
defer func() {
|
|
|
|
close(ch)
|
|
|
|
}()
|
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
for typ := range idx.byType {
|
|
|
|
m := &idx.byType[typ]
|
|
|
|
m.foreach(func(e *indexEntry) bool {
|
2016-08-03 18:03:52 +00:00
|
|
|
select {
|
2017-06-18 12:45:02 +00:00
|
|
|
case <-ctx.Done():
|
2020-06-23 20:13:25 +00:00
|
|
|
return false
|
|
|
|
case ch <- idx.toPackedBlob(e, restic.BlobType(typ)):
|
|
|
|
return true
|
2016-08-03 18:03:52 +00:00
|
|
|
}
|
2020-06-23 20:13:25 +00:00
|
|
|
})
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
return ch
|
|
|
|
}
|
|
|
|
|
2015-10-25 14:28:01 +00:00
|
|
|
// Packs returns all packs in this index
|
2016-08-31 18:29:54 +00:00
|
|
|
func (idx *Index) Packs() restic.IDSet {
|
2015-10-25 14:28:01 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2016-08-31 18:29:54 +00:00
|
|
|
packs := restic.NewIDSet()
|
2020-06-12 06:25:49 +00:00
|
|
|
for _, packID := range idx.packs {
|
|
|
|
packs.Insert(packID)
|
2015-10-25 14:28:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return packs
|
|
|
|
}
|
|
|
|
|
2015-04-26 15:44:38 +00:00
|
|
|
// Count returns the number of blobs of type t in the index.
|
2016-08-31 18:58:57 +00:00
|
|
|
func (idx *Index) Count(t restic.BlobType) (n uint) {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("counting blobs of type %v", t)
|
2015-04-26 15:44:38 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
return idx.byType[t].len()
|
2015-04-26 15:44:38 +00:00
|
|
|
}
|
|
|
|
|
2015-04-26 15:10:31 +00:00
|
|
|
type packJSON struct {
|
2016-08-31 18:29:54 +00:00
|
|
|
ID restic.ID `json:"id"`
|
2015-04-26 15:10:31 +00:00
|
|
|
Blobs []blobJSON `json:"blobs"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type blobJSON struct {
|
2016-08-31 18:58:57 +00:00
|
|
|
ID restic.ID `json:"id"`
|
|
|
|
Type restic.BlobType `json:"type"`
|
|
|
|
Offset uint `json:"offset"`
|
|
|
|
Length uint `json:"length"`
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2015-10-25 13:25:48 +00:00
|
|
|
// generatePackList returns a list of packs.
|
|
|
|
func (idx *Index) generatePackList() ([]*packJSON, error) {
|
2015-04-26 15:10:31 +00:00
|
|
|
list := []*packJSON{}
|
2016-08-31 18:29:54 +00:00
|
|
|
packs := make(map[restic.ID]*packJSON)
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
for typ := range idx.byType {
|
|
|
|
m := &idx.byType[typ]
|
|
|
|
m.foreach(func(e *indexEntry) bool {
|
|
|
|
packID := idx.packs[e.packIndex]
|
2020-06-12 06:25:49 +00:00
|
|
|
if packID.IsNull() {
|
2016-08-03 18:03:52 +00:00
|
|
|
panic("null pack id")
|
|
|
|
}
|
2015-10-11 18:45:42 +00:00
|
|
|
|
2020-06-23 20:13:25 +00:00
|
|
|
debug.Log("handle blob %v", e.id)
|
2015-04-26 15:44:38 +00:00
|
|
|
|
2016-08-03 18:03:52 +00:00
|
|
|
// see if pack is already in map
|
2020-06-12 06:25:49 +00:00
|
|
|
p, ok := packs[packID]
|
2016-08-03 18:03:52 +00:00
|
|
|
if !ok {
|
|
|
|
// else create new pack
|
2020-06-12 06:25:49 +00:00
|
|
|
p = &packJSON{ID: packID}
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2016-08-03 18:03:52 +00:00
|
|
|
// and append it to the list and map
|
|
|
|
list = append(list, p)
|
|
|
|
packs[p.ID] = p
|
|
|
|
}
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2016-08-03 18:03:52 +00:00
|
|
|
// add blob
|
|
|
|
p.Blobs = append(p.Blobs, blobJSON{
|
2020-06-23 20:13:25 +00:00
|
|
|
ID: e.id,
|
|
|
|
Type: restic.BlobType(typ),
|
|
|
|
Offset: uint(e.offset),
|
|
|
|
Length: uint(e.length),
|
2016-08-03 18:03:52 +00:00
|
|
|
})
|
2020-06-23 20:13:25 +00:00
|
|
|
|
|
|
|
return true
|
|
|
|
})
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("done")
|
2015-05-16 12:05:19 +00:00
|
|
|
|
|
|
|
return list, nil
|
|
|
|
}
|
|
|
|
|
2015-07-25 22:40:00 +00:00
|
|
|
type jsonIndex struct {
|
2016-08-31 18:29:54 +00:00
|
|
|
Supersedes restic.IDs `json:"supersedes,omitempty"`
|
2015-08-08 10:22:17 +00:00
|
|
|
Packs []*packJSON `json:"packs"`
|
2015-07-25 22:40:00 +00:00
|
|
|
}
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
// Encode writes the JSON serialization of the index to the writer w.
|
|
|
|
func (idx *Index) Encode(w io.Writer) error {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("encoding index")
|
2015-10-12 20:34:12 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
return idx.encode(w)
|
|
|
|
}
|
|
|
|
|
|
|
|
// encode writes the JSON serialization of the index to the writer w.
|
|
|
|
func (idx *Index) encode(w io.Writer) error {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("encoding index")
|
2015-10-12 20:34:12 +00:00
|
|
|
|
2015-10-25 13:25:48 +00:00
|
|
|
list, err := idx.generatePackList()
|
2015-05-16 12:05:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2015-04-26 15:10:31 +00:00
|
|
|
enc := json.NewEncoder(w)
|
2015-07-25 22:40:00 +00:00
|
|
|
idxJSON := jsonIndex{
|
2015-10-12 20:34:12 +00:00
|
|
|
Supersedes: idx.supersedes,
|
2015-07-25 22:40:00 +00:00
|
|
|
Packs: list,
|
|
|
|
}
|
|
|
|
return enc.Encode(idxJSON)
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// Finalize sets the index to final.
|
|
|
|
func (idx *Index) Finalize() {
|
|
|
|
debug.Log("finalizing index")
|
2015-05-16 12:05:19 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
idx.final = true
|
2020-06-13 10:43:03 +00:00
|
|
|
// clear packIDToIndex as no more elements will be added
|
|
|
|
idx.packIDToIndex = nil
|
2015-05-16 12:05:19 +00:00
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
// ID returns the IDs of the index, if available. If the index is not yet
|
2015-11-02 17:51:45 +00:00
|
|
|
// finalized, an error is returned.
|
2020-07-04 05:06:14 +00:00
|
|
|
func (idx *Index) IDs() (restic.IDs, error) {
|
2015-11-02 17:51:45 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
if !idx.final {
|
2020-07-04 05:06:14 +00:00
|
|
|
return nil, errors.New("index not finalized")
|
2015-11-02 17:51:45 +00:00
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
return idx.ids, nil
|
2015-11-02 17:51:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// SetID sets the ID the index has been written to. This requires that
|
|
|
|
// Finalize() has been called before, otherwise an error is returned.
|
2016-08-31 18:29:54 +00:00
|
|
|
func (idx *Index) SetID(id restic.ID) error {
|
2015-11-02 17:51:45 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
|
|
|
if !idx.final {
|
2017-02-08 23:43:10 +00:00
|
|
|
return errors.New("index is not final")
|
2015-11-02 17:51:45 +00:00
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
if len(idx.ids) > 0 {
|
2015-11-02 17:51:45 +00:00
|
|
|
return errors.New("ID already set")
|
|
|
|
}
|
|
|
|
|
2018-01-25 19:49:41 +00:00
|
|
|
debug.Log("ID set to %v", id)
|
2020-07-04 05:06:14 +00:00
|
|
|
idx.ids = append(idx.ids, id)
|
2015-11-02 17:51:45 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-05-16 12:05:19 +00:00
|
|
|
// Dump writes the pretty-printed JSON representation of the index to w.
|
|
|
|
func (idx *Index) Dump(w io.Writer) error {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("dumping index")
|
2015-05-16 12:05:19 +00:00
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
|
2015-10-25 13:25:48 +00:00
|
|
|
list, err := idx.generatePackList()
|
2015-05-16 12:05:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2015-11-02 18:28:30 +00:00
|
|
|
outer := jsonIndex{
|
|
|
|
Supersedes: idx.Supersedes(),
|
|
|
|
Packs: list,
|
|
|
|
}
|
|
|
|
|
|
|
|
buf, err := json.MarshalIndent(outer, "", " ")
|
2015-05-16 12:05:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = w.Write(append(buf, '\n'))
|
|
|
|
if err != nil {
|
2016-08-29 20:16:58 +00:00
|
|
|
return errors.Wrap(err, "Write")
|
2015-05-16 12:05:19 +00:00
|
|
|
}
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("done")
|
2015-05-16 12:05:19 +00:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-09-23 12:47:40 +00:00
|
|
|
// TreePacks returns a list of packs that contain only tree blobs.
|
|
|
|
func (idx *Index) TreePacks() restic.IDs {
|
|
|
|
return idx.treePacks
|
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
// merge() merges indexes, i.e. idx.merge(idx2) merges the contents of idx2 into idx.
|
|
|
|
// idx2 is not changed by this method.
|
|
|
|
func (idx *Index) merge(idx2 *Index) error {
|
|
|
|
idx.m.Lock()
|
|
|
|
defer idx.m.Unlock()
|
|
|
|
idx2.m.Lock()
|
|
|
|
defer idx2.m.Unlock()
|
|
|
|
|
|
|
|
if !idx2.final {
|
|
|
|
return errors.New("index to merge is not final!")
|
|
|
|
}
|
|
|
|
|
|
|
|
packlen := len(idx.packs)
|
|
|
|
// copy all index entries of idx2 to idx
|
|
|
|
for typ := range idx2.byType {
|
|
|
|
m2 := &idx2.byType[typ]
|
|
|
|
m := &idx.byType[typ]
|
|
|
|
m2.foreach(func(entry *indexEntry) bool {
|
|
|
|
// packIndex is changed as idx2.pack is appended to idx.pack, see below
|
|
|
|
m.add(entry.id, entry.packIndex+packlen, entry.offset, entry.length)
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
idx.packs = append(idx.packs, idx2.packs...)
|
|
|
|
idx.treePacks = append(idx.treePacks, idx2.treePacks...)
|
|
|
|
idx.ids = append(idx.ids, idx2.ids...)
|
|
|
|
idx.supersedes = append(idx.supersedes, idx2.supersedes...)
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-07-26 19:58:03 +00:00
|
|
|
// isErrOldIndex returns true if the error may be caused by an old index
|
|
|
|
// format.
|
|
|
|
func isErrOldIndex(err error) bool {
|
|
|
|
if e, ok := err.(*json.UnmarshalTypeError); ok && e.Value == "array" {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// ErrOldIndexFormat means an index with the old format was detected.
|
|
|
|
var ErrOldIndexFormat = errors.New("index has old format")
|
|
|
|
|
2015-04-26 15:10:31 +00:00
|
|
|
// DecodeIndex loads and unserializes an index from rd.
|
2017-01-13 21:05:34 +00:00
|
|
|
func DecodeIndex(buf []byte) (idx *Index, err error) {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("Start decoding index")
|
2017-01-13 21:05:34 +00:00
|
|
|
idxJSON := &jsonIndex{}
|
2015-07-25 22:40:00 +00:00
|
|
|
|
2017-01-13 21:05:34 +00:00
|
|
|
err = json.Unmarshal(buf, idxJSON)
|
2015-07-25 22:40:00 +00:00
|
|
|
if err != nil {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("Error %v", err)
|
2015-07-26 19:58:03 +00:00
|
|
|
|
|
|
|
if isErrOldIndex(err) {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index is probably old format, trying that")
|
2015-07-26 19:58:03 +00:00
|
|
|
err = ErrOldIndexFormat
|
|
|
|
}
|
|
|
|
|
2016-08-29 20:16:58 +00:00
|
|
|
return nil, errors.Wrap(err, "Decode")
|
2015-07-25 22:40:00 +00:00
|
|
|
}
|
|
|
|
|
2015-08-08 10:22:17 +00:00
|
|
|
idx = NewIndex()
|
2015-07-25 22:40:00 +00:00
|
|
|
for _, pack := range idxJSON.Packs {
|
2017-09-23 12:47:40 +00:00
|
|
|
var data, tree bool
|
2020-06-12 06:25:49 +00:00
|
|
|
packID := idx.addToPacks(pack.ID)
|
2017-09-23 12:47:40 +00:00
|
|
|
|
2015-07-25 22:40:00 +00:00
|
|
|
for _, blob := range pack.Blobs {
|
2020-06-12 06:25:49 +00:00
|
|
|
idx.store(packID, restic.Blob{
|
|
|
|
Type: blob.Type,
|
|
|
|
ID: blob.ID,
|
|
|
|
Offset: blob.Offset,
|
|
|
|
Length: blob.Length,
|
2015-11-02 18:05:19 +00:00
|
|
|
})
|
2017-09-23 12:47:40 +00:00
|
|
|
|
|
|
|
switch blob.Type {
|
|
|
|
case restic.DataBlob:
|
|
|
|
data = true
|
|
|
|
case restic.TreeBlob:
|
|
|
|
tree = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !data && tree {
|
|
|
|
idx.treePacks = append(idx.treePacks, pack.ID)
|
2015-07-25 22:40:00 +00:00
|
|
|
}
|
|
|
|
}
|
2015-08-08 10:22:17 +00:00
|
|
|
idx.supersedes = idxJSON.Supersedes
|
2015-10-12 20:34:12 +00:00
|
|
|
idx.final = true
|
2015-07-25 22:40:00 +00:00
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("done")
|
2016-08-29 20:16:58 +00:00
|
|
|
return idx, nil
|
2015-07-25 22:40:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// DecodeOldIndex loads and unserializes an index in the old format from rd.
|
2017-01-13 21:05:34 +00:00
|
|
|
func DecodeOldIndex(buf []byte) (idx *Index, err error) {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("Start decoding old index")
|
2015-04-26 15:10:31 +00:00
|
|
|
list := []*packJSON{}
|
|
|
|
|
2017-01-13 21:05:34 +00:00
|
|
|
err = json.Unmarshal(buf, &list)
|
2015-04-26 15:10:31 +00:00
|
|
|
if err != nil {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("Error %#v", err)
|
2016-08-29 20:16:58 +00:00
|
|
|
return nil, errors.Wrap(err, "Decode")
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
|
2015-08-08 10:22:17 +00:00
|
|
|
idx = NewIndex()
|
2015-04-26 15:10:31 +00:00
|
|
|
for _, pack := range list {
|
2017-09-23 12:47:40 +00:00
|
|
|
var data, tree bool
|
2020-06-12 06:25:49 +00:00
|
|
|
packID := idx.addToPacks(pack.ID)
|
2017-09-23 12:47:40 +00:00
|
|
|
|
2015-04-26 15:10:31 +00:00
|
|
|
for _, blob := range pack.Blobs {
|
2020-06-12 06:25:49 +00:00
|
|
|
idx.store(packID, restic.Blob{
|
|
|
|
Type: blob.Type,
|
|
|
|
ID: blob.ID,
|
|
|
|
Offset: blob.Offset,
|
|
|
|
Length: blob.Length,
|
2015-11-02 18:05:19 +00:00
|
|
|
})
|
2017-09-23 12:47:40 +00:00
|
|
|
|
|
|
|
switch blob.Type {
|
|
|
|
case restic.DataBlob:
|
|
|
|
data = true
|
|
|
|
case restic.TreeBlob:
|
|
|
|
tree = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !data && tree {
|
|
|
|
idx.treePacks = append(idx.treePacks, pack.ID)
|
2015-04-26 15:10:31 +00:00
|
|
|
}
|
|
|
|
}
|
2015-11-08 21:21:29 +00:00
|
|
|
idx.final = true
|
2015-04-26 15:10:31 +00:00
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("done")
|
2016-08-29 20:16:58 +00:00
|
|
|
return idx, nil
|
2015-08-08 10:22:17 +00:00
|
|
|
}
|
|
|
|
|
2015-11-02 17:51:24 +00:00
|
|
|
// LoadIndexWithDecoder loads the index and decodes it with fn.
|
2019-03-24 21:12:38 +00:00
|
|
|
func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, buf []byte, id restic.ID, fn func([]byte) (*Index, error)) (*Index, []byte, error) {
|
2018-01-25 19:49:41 +00:00
|
|
|
debug.Log("Loading index %v", id)
|
2015-11-02 17:51:24 +00:00
|
|
|
|
2019-03-24 21:12:38 +00:00
|
|
|
buf, err := repo.LoadAndDecrypt(ctx, buf[:0], restic.IndexFile, id)
|
2015-11-02 17:51:24 +00:00
|
|
|
if err != nil {
|
2019-03-24 21:12:38 +00:00
|
|
|
return nil, buf[:0], err
|
2015-11-02 17:51:24 +00:00
|
|
|
}
|
|
|
|
|
2019-03-24 21:12:38 +00:00
|
|
|
idx, err := fn(buf)
|
2015-11-02 17:51:24 +00:00
|
|
|
if err != nil {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("error while decoding index %v: %v", id, err)
|
2019-03-24 21:12:38 +00:00
|
|
|
return nil, buf[:0], err
|
2015-11-02 17:51:24 +00:00
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
idx.ids = append(idx.ids, id)
|
2015-11-02 17:51:24 +00:00
|
|
|
|
2019-03-24 21:12:38 +00:00
|
|
|
return idx, buf, nil
|
2015-11-02 17:51:24 +00:00
|
|
|
}
|