From 837b8163581b26cdf6295fbf719f87b6e2a80297 Mon Sep 17 00:00:00 2001 From: plumbeo Date: Mon, 5 Sep 2022 14:18:12 +0200 Subject: [PATCH 1/4] restic stats: print uncompressed size in mode raw-data --- cmd/restic/cmd_stats.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a8bcb2b85..5f089d926 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -135,6 +135,13 @@ func runStats(gopts GlobalOptions, args []string) error { return fmt.Errorf("blob %v not found", blobHandle) } stats.TotalSize += uint64(pbs[0].Length) + if repo.Config().Version >= 2 { + if pbs[0].IsCompressed() { + stats.TotalUncompressedSize += uint64(pbs[0].UncompressedLength) + } else { + stats.TotalUncompressedSize += uint64(pbs[0].Length) + } + } stats.TotalBlobCount++ } } @@ -148,15 +155,18 @@ func runStats(gopts GlobalOptions, args []string) error { } Printf("Stats in %s mode:\n", statsOptions.countMode) - Printf("Snapshots processed: %d\n", stats.SnapshotsCount) + Printf(" Snapshots processed: %d\n", stats.SnapshotsCount) if stats.TotalBlobCount > 0 { - Printf(" Total Blob Count: %d\n", stats.TotalBlobCount) + Printf(" Total Blob Count: %d\n", stats.TotalBlobCount) } if stats.TotalFileCount > 0 { - Printf(" Total File Count: %d\n", stats.TotalFileCount) + Printf(" Total File Count: %d\n", stats.TotalFileCount) } - Printf(" Total Size: %-5s\n", formatBytes(stats.TotalSize)) + if stats.TotalUncompressedSize > 0 { + Printf("Total Uncompressed Size: %-5s\n", formatBytes(stats.TotalUncompressedSize)) + } + Printf(" Total Size: %-5s\n", formatBytes(stats.TotalSize)) return nil } @@ -282,9 +292,10 @@ func verifyStatsInput(gopts GlobalOptions, args []string) error { // to collect information about it, as well as state needed // for a successful and efficient walk. type statsContainer struct { - TotalSize uint64 `json:"total_size"` - TotalFileCount uint64 `json:"total_file_count"` - TotalBlobCount uint64 `json:"total_blob_count,omitempty"` + TotalSize uint64 `json:"total_size"` + TotalUncompressedSize uint64 `json:"total_uncompressed_size"` + TotalFileCount uint64 `json:"total_file_count"` + TotalBlobCount uint64 `json:"total_blob_count,omitempty"` // holds count of all considered snapshots SnapshotsCount int `json:"snapshots_count"` From d66e755ac70f3a11170e43e41236672a08f3f9a2 Mon Sep 17 00:00:00 2001 From: plumbeo Date: Wed, 7 Sep 2022 10:09:59 +0200 Subject: [PATCH 2/4] Change uncompressed size calculation to account for the encryption overhead --- cmd/restic/cmd_stats.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 5f089d926..d74ec6a10 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -136,11 +137,7 @@ func runStats(gopts GlobalOptions, args []string) error { } stats.TotalSize += uint64(pbs[0].Length) if repo.Config().Version >= 2 { - if pbs[0].IsCompressed() { - stats.TotalUncompressedSize += uint64(pbs[0].UncompressedLength) - } else { - stats.TotalUncompressedSize += uint64(pbs[0].Length) - } + stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) } stats.TotalBlobCount++ } @@ -293,7 +290,7 @@ func verifyStatsInput(gopts GlobalOptions, args []string) error { // for a successful and efficient walk. type statsContainer struct { TotalSize uint64 `json:"total_size"` - TotalUncompressedSize uint64 `json:"total_uncompressed_size"` + TotalUncompressedSize uint64 `json:"total_uncompressed_size,omitempty"` TotalFileCount uint64 `json:"total_file_count"` TotalBlobCount uint64 `json:"total_blob_count,omitempty"` // holds count of all considered snapshots From bc945d0bf0512c1fa7dd901b5c5a786c32b2eada Mon Sep 17 00:00:00 2001 From: plumbeo Date: Tue, 11 Oct 2022 14:36:12 +0200 Subject: [PATCH 3/4] restic stats: add more compression statistics Calculate and display compression ratio, space saving and progress --- cmd/restic/cmd_stats.go | 44 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index d74ec6a10..7db3e5aef 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -138,9 +138,20 @@ func runStats(gopts GlobalOptions, args []string) error { stats.TotalSize += uint64(pbs[0].Length) if repo.Config().Version >= 2 { stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + if pbs[0].IsCompressed() { + stats.TotalCompressedBlobsSize += uint64(pbs[0].Length) + stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + } } stats.TotalBlobCount++ } + if stats.TotalCompressedBlobsSize > 0 { + stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize) + } + if stats.TotalUncompressedSize > 0 { + stats.CompressionProgress = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalUncompressedSize) * 100 + stats.CompressionSpaceSaving = (1 - float64(stats.TotalSize)/float64(stats.TotalUncompressedSize)) * 100 + } } if gopts.JSON { @@ -152,18 +163,26 @@ func runStats(gopts GlobalOptions, args []string) error { } Printf("Stats in %s mode:\n", statsOptions.countMode) - Printf(" Snapshots processed: %d\n", stats.SnapshotsCount) - + Printf(" Snapshots processed: %d\n", stats.SnapshotsCount) if stats.TotalBlobCount > 0 { - Printf(" Total Blob Count: %d\n", stats.TotalBlobCount) + Printf(" Total Blob Count: %d\n", stats.TotalBlobCount) } if stats.TotalFileCount > 0 { - Printf(" Total File Count: %d\n", stats.TotalFileCount) + Printf(" Total File Count: %d\n", stats.TotalFileCount) } if stats.TotalUncompressedSize > 0 { - Printf("Total Uncompressed Size: %-5s\n", formatBytes(stats.TotalUncompressedSize)) + Printf(" Total Uncompressed Size: %-5s\n", formatBytes(stats.TotalUncompressedSize)) + } + Printf(" Total Size: %-5s\n", formatBytes(stats.TotalSize)) + if stats.CompressionProgress > 0 { + Printf(" Compression Progress: %.2f%%\n", stats.CompressionProgress) + } + if stats.CompressionRatio > 0 { + Printf(" Compression Ratio: %.2fx\n", stats.CompressionRatio) + } + if stats.CompressionSpaceSaving > 0 { + Printf("Compression Space Saving: %.2f%%\n", stats.CompressionSpaceSaving) } - Printf(" Total Size: %-5s\n", formatBytes(stats.TotalSize)) return nil } @@ -289,10 +308,15 @@ func verifyStatsInput(gopts GlobalOptions, args []string) error { // to collect information about it, as well as state needed // for a successful and efficient walk. type statsContainer struct { - TotalSize uint64 `json:"total_size"` - TotalUncompressedSize uint64 `json:"total_uncompressed_size,omitempty"` - TotalFileCount uint64 `json:"total_file_count"` - TotalBlobCount uint64 `json:"total_blob_count,omitempty"` + TotalSize uint64 `json:"total_size"` + TotalUncompressedSize uint64 `json:"total_uncompressed_size,omitempty"` + TotalCompressedBlobsSize uint64 `json:"-"` + TotalCompressedBlobsUncompressedSize uint64 `json:"-"` + CompressionRatio float64 `json:"compression_ratio,omitempty"` + CompressionProgress float64 `json:"compression_progress,omitempty"` + CompressionSpaceSaving float64 `json:"compression_space_saving,omitempty"` + TotalFileCount uint64 `json:"total_file_count,omitempty"` + TotalBlobCount uint64 `json:"total_blob_count,omitempty"` // holds count of all considered snapshots SnapshotsCount int `json:"snapshots_count"` From a6f83e0011239c064fc8632b74f1c8c5c3450792 Mon Sep 17 00:00:00 2001 From: plumbeo Date: Tue, 11 Oct 2022 14:55:32 +0200 Subject: [PATCH 4/4] Add changelog --- changelog/unreleased/pull-3915 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/unreleased/pull-3915 diff --git a/changelog/unreleased/pull-3915 b/changelog/unreleased/pull-3915 new file mode 100644 index 000000000..3b966a425 --- /dev/null +++ b/changelog/unreleased/pull-3915 @@ -0,0 +1,9 @@ +Enhancement: add compression statistics to the restic stats command + +When executed in `raw-data` mode on a repository that supports compression, the `restic stats` +command now calculates and displays, for the selected repository or snapshots, the uncompressed +size of the data, the compression progress (percentage of data that has been compressed), the +compression ratio of the compressed data and the total space saving, taking into account both +the compressed and uncompressed data if the repository is only partially compressed. + +https://github.com/restic/restic/pull/3915