adapt workers based on whether an operation is CPU or IO-bound
Use runtime.GOMAXPROCS(0) as worker count for CPU-bound tasks, repo.Connections() for IO-bound task and a combination if a task can be both. Streaming packs is treated as IO-bound as adding more worker cannot provide a speedup. Typical IO-bound tasks are download / uploading / deleting files. Decoding / Encoding / Verifying are usually CPU-bound. Several tasks are a combination of both, e.g. for combined download and decode functions. In the latter case add both limits together. As the backends have their own concurrency limits restic still won't download more than repo.Connections() files in parallel, but the additional workers can decode already downloaded data in parallel.
This commit is contained in:
parent
cd50feb66f
commit
6f53ecc1ae
16 changed files with 66 additions and 40 deletions
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
@ -291,8 +292,6 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
const saveIndexParallelism = 4
|
||||
|
||||
// Save saves all known indexes to index files, leaving out any
|
||||
// packs whose ID is contained in packBlacklist from finalized indexes.
|
||||
// The new index contains the IDs of all known indexes in the "supersedes"
|
||||
|
@ -376,8 +375,10 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack
|
|||
return nil
|
||||
}
|
||||
|
||||
// encoding an index can take quite some time such that this can be both CPU- or IO-bound
|
||||
workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0)
|
||||
// run workers on ch
|
||||
for i := 0; i < saveIndexParallelism; i++ {
|
||||
for i := 0; i < workerCount; i++ {
|
||||
wg.Go(worker)
|
||||
}
|
||||
err = wg.Wait()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue