restic/internal/restorer/filerestorer.go

package restorer

import (
	"context"
	"io"
	"os"

	"github.com/restic/restic/internal/crypto"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
)

// TODO if a blob is corrupt, there may be good blob copies in other packs
// TODO evaluate if it makes sense to split download and processing workers
//      pro: can (slowly) read network and decrypt/write files concurrently
//      con: each worker needs to keep one pack in memory
// TODO evaluate memory footprint for larger repositories, say 10M packs/10M files
// TODO consider replacing pack file cache with blob cache
// TODO avoid decrypting the same blob multiple times
// TODO evaluate disabled debug logging overhead for large repositories
// TODO consider logging snapshot-relative path to reduce log clutter

const (
	workerCount = 8

	// max number of open output file handles
	filesWriterCount = 32

	// estimated average pack size used to calculate pack cache capacity
	averagePackSize = 5 * 1024 * 1024

	// pack cache capacity should support at least one cached pack per worker
	// allow space for extra 5 packs for actual caching
	packCacheCapacity = (workerCount + 5) * averagePackSize
)

// information about regular file being restored
type fileInfo struct {
	path  string      // full path to the file on local filesystem
	blobs []restic.ID // remaining blobs of the file
}

// information about a data pack required to restore one or more files
type packInfo struct {
	// the pack id
	id restic.ID

	// set of files that use blobs from this pack
	files map[*fileInfo]struct{}

	// number of other packs that must be downloaded before all blobs in this pack can be used
	cost int

	// used by packHeap
	index int
}

// fileRestorer restores set of files
type fileRestorer struct {
	key        *crypto.Key
	idx        filePackTraverser
	packLoader func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error

	packCache   *packCache   // pack cache
	filesWriter *filesWriter // file write

	files []*fileInfo
}

func newFileRestorer(packLoader func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error, key *crypto.Key, idx filePackTraverser) *fileRestorer {
	return &fileRestorer{
		packLoader:  packLoader,
		key:         key,
		idx:         idx,
		filesWriter: newFilesWriter(filesWriterCount),
		packCache:   newPackCache(packCacheCapacity),
	}
}

func (r *fileRestorer) addFile(path string, content restic.IDs) {
	r.files = append(r.files, &fileInfo{path: path, blobs: content})
}

// used to pass information among workers (wish golang channels allowed multivalues)
type processingInfo struct {
	pack  *packInfo
	files map[*fileInfo]error
}

func (r *fileRestorer) restoreFiles(ctx context.Context, onError func(path string, err error)) error {
	for _, file := range r.files {
		dbgmsg := file.path + ": "
		r.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
			if packIdx > 0 {
				dbgmsg += ", "
			}
			dbgmsg += "pack{id=" + packID.Str() + ", blobs: "
			for blobIdx, blob := range packBlobs {
				if blobIdx > 0 {
					dbgmsg += ", "
				}
				dbgmsg += blob.ID.Str()
			}
			dbgmsg += "}"
			return true // keep going
		})
		debug.Log(dbgmsg)
	}

	// synchronously create empty files (empty files need no packs and are ignored by packQueue)
	for _, file := range r.files {
		if len(file.blobs) == 0 {
			wr, err := os.OpenFile(file.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
			if err == nil {
				wr.Close()
			} else {
				onError(file.path, err)
			}
		}
	}

	inprogress := make(map[*fileInfo]struct{})
	queue, err := newPackQueue(r.idx, r.files, func(files map[*fileInfo]struct{}) bool {
		for file := range files {
			if _, found := inprogress[file]; found {
				return true
			}
		}
		return false
	})
	if err != nil {
		return err
	}

	// workers
	downloadCh := make(chan processingInfo)
	feedbackCh := make(chan processingInfo)

	defer close(downloadCh)
	defer close(feedbackCh)

	worker := func() {
		for {
			select {
			case <-ctx.Done():
				return
			case request, ok := <-downloadCh:
				if !ok {
					return // channel closed
				}
				rd, err := r.downloadPack(ctx, request.pack)
				if err == nil {
					r.processPack(ctx, request, rd)
				} else {
					// mark all files as failed
					for file := range request.files {
						request.files[file] = err
					}
				}
				feedbackCh <- request
			}
		}
	}
	for i := 0; i < workerCount; i++ {
		go worker()
	}

	processFeedback := func(pack *packInfo, ferrors map[*fileInfo]error) {
		// update files blobIdx
		// must do it here to avoid race among worker and processing feedback threads
		var success []*fileInfo
		var failure []*fileInfo
		for file, ferr := range ferrors {
			if ferr != nil {
				onError(file.path, ferr)
				r.filesWriter.close(file)
				delete(inprogress, file)
				failure = append(failure, file)
			} else {
				r.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
					file.blobs = file.blobs[len(packBlobs):]
					return false // only interesed in the first pack
				})
				if len(file.blobs) == 0 {
					r.filesWriter.close(file)
					delete(inprogress, file)
				}
				success = append(success, file)
			}
		}
		// update the queue and requeueu the pack as necessary
		if !queue.requeuePack(pack, success, failure) {
			r.packCache.remove(pack.id)
			debug.Log("Purged used up pack %s from pack cache", pack.id.Str())
		}
	}

	// the main restore loop
	for !queue.isEmpty() {
		debug.Log("-----------------------------------")
		pack, files := queue.nextPack()
		if pack != nil {
			ferrors := make(map[*fileInfo]error)
			for _, file := range files {
				ferrors[file] = nil
				inprogress[file] = struct{}{}
			}
			select {
			case <-ctx.Done():
				return ctx.Err()
			case downloadCh <- processingInfo{pack: pack, files: ferrors}:
				debug.Log("Scheduled download pack %s (%d files)", pack.id.Str(), len(files))
			case feedback := <-feedbackCh:
				queue.requeuePack(pack, []*fileInfo{}, []*fileInfo{}) // didn't use the pack during this iteration
				processFeedback(feedback.pack, feedback.files)
			}
		} else {
			select {
			case <-ctx.Done():
				return ctx.Err()
			case feedback := <-feedbackCh:
				processFeedback(feedback.pack, feedback.files)
			}
		}
	}

	return nil
}

func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) (readerAtCloser, error) {
	const MaxInt64 = 1<<63 - 1 // odd Go does not have this predefined somewhere

	// calculate pack byte range
	start, end := int64(MaxInt64), int64(0)
	for file := range pack.files {
		r.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
			if packID.Equal(pack.id) {
				for _, blob := range packBlobs {
					if start > int64(blob.Offset) {
						start = int64(blob.Offset)
					}
					if end < int64(blob.Offset+blob.Length) {
						end = int64(blob.Offset + blob.Length)
					}
				}
			}

			return true // keep going
		})
	}

	packReader, err := r.packCache.get(pack.id, start, int(end-start), func(offset int64, length int, wr io.WriteSeeker) error {
		h := restic.Handle{Type: restic.DataFile, Name: pack.id.String()}
		return r.packLoader(ctx, h, length, offset, func(rd io.Reader) error {
			// reset the file in case of a download retry
			_, err := wr.Seek(0, io.SeekStart)
			if err != nil {
				return err
			}

			len, err := io.Copy(wr, rd)
			if err != nil {
				return err
			}
			if len != int64(length) {
				return errors.Errorf("unexpected pack size: expected %d but got %d", length, len)
			}

			return nil
		})
	})
	if err != nil {
		return nil, err
	}

	return packReader, nil
}

func (r *fileRestorer) processPack(ctx context.Context, request processingInfo, rd readerAtCloser) {
	defer rd.Close()

	for file := range request.files {
		r.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
			for _, blob := range packBlobs {
				debug.Log("Writing blob %s (%d bytes) from pack %s to %s", blob.ID.Str(), blob.Length, packID.Str(), file.path)
				buf, err := r.loadBlob(rd, blob)
				if err == nil {
					err = r.filesWriter.writeToFile(file, buf)
				}
				if err != nil {
					request.files[file] = err
					break // could not restore the file
				}
			}
			return false
		})
	}
}

func (r *fileRestorer) loadBlob(rd io.ReaderAt, blob restic.Blob) ([]byte, error) {
	// TODO reconcile with Repository#loadBlob implementation

	buf := make([]byte, blob.Length)

	n, err := rd.ReadAt(buf, int64(blob.Offset))
	if err != nil {
		return nil, err
	}

	if n != int(blob.Length) {
		return nil, errors.Errorf("error loading blob %v: wrong length returned, want %d, got %d", blob.ID.Str(), blob.Length, n)
	}

	// decrypt
	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
	plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil)
	if err != nil {
		return nil, errors.Errorf("decrypting blob %v failed: %v", blob.ID, err)
	}

	// check hash
	if !restic.Hash(plaintext).Equal(blob.ID) {
		return nil, errors.Errorf("blob %v returned invalid hash", blob.ID)
	}

	return plaintext, nil
}