From e388d962a5d4608a583687c7afe1e86ea2539894 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 4 Jul 2020 07:06:14 +0200 Subject: [PATCH] Merge final indexes together for faster index access --- internal/checker/checker.go | 2 ++ internal/repository/index.go | 50 +++++++++++++++++++++++------ internal/repository/index_test.go | 5 ++- internal/repository/master_index.go | 38 +++++++++++++++++++--- internal/repository/repository.go | 15 ++++++--- 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 2e2e233d6..185fb52ce 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -190,6 +190,8 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { } } + c.masterIndex.MergeFinalIndexes() + err = c.repo.SetIndex(c.masterIndex) if err != nil { debug.Log("SetIndex returned error: %v", err) diff --git a/internal/repository/index.go b/internal/repository/index.go index 560696ba5..e6053fa72 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -49,8 +49,8 @@ type Index struct { // only used by Store, StorePacks does not check for already saved packIDs packIDToIndex map[restic.ID]int - final bool // set to true for all indexes read from the backend ("finalized") - id restic.ID // set to the ID of the index when it's finalized + final bool // set to true for all indexes read from the backend ("finalized") + ids restic.IDs // set to the IDs of the contained finalized indexes supersedes restic.IDs created time.Time } @@ -393,17 +393,17 @@ func (idx *Index) Finalize() { idx.packIDToIndex = nil } -// ID returns the ID of the index, if available. If the index is not yet +// ID returns the IDs of the index, if available. If the index is not yet // finalized, an error is returned. -func (idx *Index) ID() (restic.ID, error) { +func (idx *Index) IDs() (restic.IDs, error) { idx.m.Lock() defer idx.m.Unlock() if !idx.final { - return restic.ID{}, errors.New("index not finalized") + return nil, errors.New("index not finalized") } - return idx.id, nil + return idx.ids, nil } // SetID sets the ID the index has been written to. This requires that @@ -416,12 +416,12 @@ func (idx *Index) SetID(id restic.ID) error { return errors.New("index is not final") } - if !idx.id.IsNull() { + if len(idx.ids) > 0 { return errors.New("ID already set") } debug.Log("ID set to %v", id) - idx.id = id + idx.ids = append(idx.ids, id) return nil } @@ -462,6 +462,38 @@ func (idx *Index) TreePacks() restic.IDs { return idx.treePacks } +// merge() merges indexes, i.e. idx.merge(idx2) merges the contents of idx2 into idx. +// idx2 is not changed by this method. +func (idx *Index) merge(idx2 *Index) error { + idx.m.Lock() + defer idx.m.Unlock() + idx2.m.Lock() + defer idx2.m.Unlock() + + if !idx2.final { + return errors.New("index to merge is not final!") + } + + packlen := len(idx.packs) + // copy all index entries of idx2 to idx + for typ := range idx2.byType { + m2 := &idx2.byType[typ] + m := &idx.byType[typ] + m2.foreach(func(entry *indexEntry) bool { + // packIndex is changed as idx2.pack is appended to idx.pack, see below + m.add(entry.id, entry.packIndex+packlen, entry.offset, entry.length) + return true + }) + } + + idx.packs = append(idx.packs, idx2.packs...) + idx.treePacks = append(idx.treePacks, idx2.treePacks...) + idx.ids = append(idx.ids, idx2.ids...) + idx.supersedes = append(idx.supersedes, idx2.supersedes...) + + return nil +} + // isErrOldIndex returns true if the error may be caused by an old index // format. func isErrOldIndex(err error) bool { @@ -581,7 +613,7 @@ func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, buf []byt return nil, buf[:0], err } - idx.id = id + idx.ids = append(idx.ids, id) return idx, buf, nil } diff --git a/internal/repository/index_test.go b/internal/repository/index_test.go index 07328058c..7b95a13c2 100644 --- a/internal/repository/index_test.go +++ b/internal/repository/index_test.go @@ -135,10 +135,9 @@ func TestIndexSerialize(t *testing.T) { id := restic.NewRandomID() rtest.OK(t, idx.SetID(id)) - id2, err := idx.ID() + ids, err := idx.IDs() rtest.OK(t, err) - rtest.Assert(t, id2.Equal(id), - "wrong ID returned: want %v, got %v", id, id2) + rtest.Equals(t, restic.IDs{id}, ids) idx3, err := repository.DecodeIndex(wr3.Bytes()) rtest.OK(t, err) diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 24762ffb2..2705b5994 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -18,7 +18,12 @@ type MasterIndex struct { // NewMasterIndex creates a new master index. func NewMasterIndex() *MasterIndex { - return &MasterIndex{pendingBlobs: restic.NewBlobSet()} + // Always add an empty final index, such that MergeFinalIndexes can merge into this. + // Note that removing this index could lead to a race condition in the rare + // sitation that only two indexes exist which are saved and merged concurrently. + idx := []*Index{NewIndex()} + idx[0].Finalize() + return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} } // Lookup queries all known Indexes for the ID and returns the first match. @@ -237,6 +242,31 @@ func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob { return ch } +// MergeFinalIndexes merges all final indexes together. +// After calling, there will be only one big final index in MasterIndex +// containing all final index contents. +// Indexes that are not final are left untouched. +// This merging can only be called after all index files are loaded - as +// removing of superseded index contents is only possible for unmerged indexes. +func (mi *MasterIndex) MergeFinalIndexes() { + mi.idxMutex.Lock() + defer mi.idxMutex.Unlock() + + // The first index is always final and the one to merge into + newIdx := mi.idx[:1] + for i := 1; i < len(mi.idx); i++ { + idx := mi.idx[i] + // clear reference in masterindex as it may become stale + mi.idx[i] = nil + if !idx.Final() { + newIdx = append(newIdx, idx) + } else { + mi.idx[0].merge(idx) + } + } + mi.idx = newIdx +} + // RebuildIndex combines all known indexes to a new index, leaving out any // packs whose ID is contained in packBlacklist. The new index contains the IDs // of all known indexes in the "supersedes" field. @@ -267,15 +297,15 @@ func (mi *MasterIndex) RebuildIndex(packBlacklist restic.IDSet) (*Index, error) continue } - id, err := idx.ID() + ids, err := idx.IDs() if err != nil { debug.Log("index %d does not have an ID: %v", err) return nil, err } - debug.Log("adding index id %v to supersedes field", id) + debug.Log("adding index ids %v to supersedes field", ids) - err = newIndex.AddToSupersedes(id) + err = newIndex.AddToSupersedes(ids...) if err != nil { return nil, err } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 64781f9e4..0d5242022 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -361,12 +361,14 @@ func (r *Repository) SetIndex(i restic.Index) error { ids := restic.NewIDSet() for _, idx := range r.idx.All() { - id, err := idx.ID() + indexIDs, err := idx.IDs() if err != nil { debug.Log("not using index, ID() returned error %v", err) continue } - ids.Insert(id) + for _, id := range indexIDs { + ids.Insert(id) + } } return r.PrepareCache(ids) @@ -396,6 +398,7 @@ func (r *Repository) saveIndex(ctx context.Context, indexes ...*Index) error { debug.Log("Saved index %d as %v", i, sid) } + r.idx.MergeFinalIndexes() return nil } @@ -479,12 +482,16 @@ func (r *Repository) LoadIndex(ctx context.Context) error { validIndex := restic.NewIDSet() wg.Go(func() error { for idx := range indexCh { - id, err := idx.ID() + ids, err := idx.IDs() if err == nil { - validIndex.Insert(id) + for _, id := range ids { + validIndex.Insert(id) + } } + r.idx.Insert(idx) } + r.idx.MergeFinalIndexes() return nil })