Fix O(n) backend list calls in FindFilteredSnapshots

When resolving snapshotIDs in FindFilteredSnapshots either
FindLatestSnapshot or FindSnapshot is called. Both operations issue a
list operation to the backend. When for example passing a long list of
snapshot ids to `forget` this could lead to a large number of list
operations.
This commit is contained in:
Michael Eischer 2021-11-06 01:23:12 +01:00
parent 3d29083e60
commit 9e12159230
2 changed files with 12 additions and 1 deletions

View file

@ -3,6 +3,7 @@ package main
import ( import (
"context" "context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
@ -12,10 +13,16 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.
go func() { go func() {
defer close(out) defer close(out)
if len(snapshotIDs) != 0 { if len(snapshotIDs) != 0 {
// memorize snapshots list to prevent repeated backend listings
be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
if err != nil {
Warnf("could not load snapshots: %v\n", err)
return
}
var ( var (
id restic.ID id restic.ID
usedFilter bool usedFilter bool
err error
) )
ids := make(restic.IDs, 0, len(snapshotIDs)) ids := make(restic.IDs, 0, len(snapshotIDs))
// Process all snapshot IDs given as arguments. // Process all snapshot IDs given as arguments.

View file

@ -81,6 +81,10 @@ func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(r
} }
func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []restic.FileInfo var fileInfos []restic.FileInfo
err := be.List(ctx, t, func(fi restic.FileInfo) error { err := be.List(ctx, t, func(fi restic.FileInfo) error {
fileInfos = append(fileInfos, fi) fileInfos = append(fileInfos, fi)