From f1b73c9301bd35388f058dcd202358ce1c85a40b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:51:50 +0200 Subject: [PATCH 1/4] Reduce GOGC to 50 The index used by restic consumes a major part of the total memory usage. This leads to an unnecessarily large amount of memory that contains ephemeral objects that are only used for a short time. --- cmd/restic/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 392177d13..64b75b43a 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -7,6 +7,7 @@ import ( "log" "os" "runtime" + godebug "runtime/debug" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/options" @@ -81,7 +82,16 @@ func needsPassword(cmd string) bool { var logBuffer = bytes.NewBuffer(nil) +func tweakGoGC() { + // lower GOGC from 100 to 50, unless it was manually overwritten by the user + oldValue := godebug.SetGCPercent(50) + if oldValue != 100 { + godebug.SetGCPercent(oldValue) + } +} + func main() { + tweakGoGC() // install custom global logger into a buffer, if an error occurs // we can show the logs log.SetOutput(logBuffer) From eef0ee7a85994e4696368e39d8f7d13e07017115 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:56:14 +0200 Subject: [PATCH 2/4] repository: trigger GC after loading the index Loading the index requires some scratch space, thus make sure that this memory does not factor into the targeted gc memory usage limit. --- internal/repository/repository.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 9d1b40c64..653c1f774 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "runtime" "sort" "sync" @@ -601,6 +602,9 @@ func (r *Repository) LoadIndex(ctx context.Context) error { return err } + // Trigger GC to reset garbage collection threshold + runtime.GC() + if r.cfg.Version < 2 { // sanity check ctx, cancel := context.WithCancel(ctx) From 2fcb3947df77427cafdb613acd831d9cbc111530 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Jun 2023 21:57:40 +0200 Subject: [PATCH 3/4] prune: trigger GC after prune planning --- cmd/restic/cmd_prune.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 26f21b1f3..1889dffd6 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -3,6 +3,7 @@ package main import ( "context" "math" + "runtime" "sort" "strconv" "strings" @@ -205,6 +206,9 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption return err } + // Trigger GC to reset garbage collection threshold + runtime.GC() + return doPrune(ctx, opts, gopts, repo, plan) } From 0f80b6a137e0ffd79dce0887be457d62c0c826db Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 18:02:46 +0200 Subject: [PATCH 4/4] add changelog for gc tuning --- changelog/unreleased/issue-3328 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/issue-3328 diff --git a/changelog/unreleased/issue-3328 b/changelog/unreleased/issue-3328 new file mode 100644 index 000000000..a8ef76d79 --- /dev/null +++ b/changelog/unreleased/issue-3328 @@ -0,0 +1,5 @@ +Enhancement: Reduce memory usage by up to 25% + +https://github.com/restic/restic/issues/3328 +https://github.com/restic/restic/pull/4352 +https://github.com/restic/restic/pull/4353