forked from TrueCloudLab/restic
120ccc8754
Previously, SaveAndEncrypt would assemble blobs into packs and either return immediately if the pack is not yet full or upload the pack file otherwise. The upload will block the current goroutine until it finishes. Now, the upload is done using separate goroutines. This requires changes to the error handling. As uploads are no longer tied to a SaveAndEncrypt call, failed uploads are signaled using an errgroup. To count the uploaded amount of data, the pack header overhead is no longer returned by `packer.Finalize` but rather by `packer.HeaderOverhead`. This helper method is necessary to continue returning the pack header overhead directly to the responsible call to `repository.SaveBlob`. Without the method this would not be possible, as packs are finalized asynchronously.
63 lines
1.2 KiB
Go
63 lines
1.2 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/restic/restic/internal/restic"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// SavePacker implements saving a pack in the repository.
|
|
type SavePacker interface {
|
|
savePacker(ctx context.Context, t restic.BlobType, p *Packer) error
|
|
}
|
|
|
|
type uploadTask struct {
|
|
packer *Packer
|
|
tpe restic.BlobType
|
|
}
|
|
|
|
type packerUploader struct {
|
|
uploadQueue chan uploadTask
|
|
}
|
|
|
|
func newPackerUploader(ctx context.Context, wg *errgroup.Group, repo SavePacker, connections uint) *packerUploader {
|
|
pu := &packerUploader{
|
|
uploadQueue: make(chan uploadTask),
|
|
}
|
|
|
|
for i := 0; i < int(connections); i++ {
|
|
wg.Go(func() error {
|
|
for {
|
|
select {
|
|
case t, ok := <-pu.uploadQueue:
|
|
if !ok {
|
|
return nil
|
|
}
|
|
err := repo.savePacker(ctx, t.tpe, t.packer)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
return pu
|
|
}
|
|
|
|
func (pu *packerUploader) QueuePacker(ctx context.Context, t restic.BlobType, p *Packer) (err error) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case pu.uploadQueue <- uploadTask{tpe: t, packer: p}:
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (pu *packerUploader) TriggerShutdown() {
|
|
close(pu.uploadQueue)
|
|
}
|