From 894ec9d05d56c3de7ce124832797d1657b98077e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jul 2024 13:08:41 +0200 Subject: [PATCH 1/4] ls: fix broken folder if --ncdu an file filters are combined --- cmd/restic/cmd_ls.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index c4fb32de3..7499da68f 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -367,9 +367,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return nil } + printedDir := false if withinDir(nodepath) { // if we're within a dir, print the node printer.Node(nodepath, node) + printedDir = true // if recursive listing is requested, signal the walker that it // should continue walking recursively @@ -387,6 +389,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri // otherwise, signal the walker to not walk recursively into any // subdirs if node.Type == "dir" { + if printedDir { + printer.LeaveDir(nodepath) + } return walker.ErrSkipNode } return nil From 15419d603dbfa0159963331ad1a7168d35ba5871 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jul 2024 13:28:56 +0200 Subject: [PATCH 2/4] ls: add missing intermediate directories to --ncdu output --- cmd/restic/cmd_ls.go | 26 ++++++++++++++++++-------- cmd/restic/cmd_ls_integration_test.go | 15 +++++++++------ cmd/restic/cmd_ls_test.go | 4 ++-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 7499da68f..8dd2af2d6 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -71,7 +71,7 @@ func init() { type lsPrinter interface { Snapshot(sn *restic.Snapshot) - Node(path string, node *restic.Node) + Node(path string, node *restic.Node, isPrefixDirectory bool) LeaveDir(path string) Close() } @@ -102,7 +102,10 @@ func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { } // Print node in our custom JSON format, followed by a newline. -func (p *jsonLsPrinter) Node(path string, node *restic.Node) { +func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) { + if isPrefixDirectory { + return + } err := lsNodeJSON(p.enc, path, node) if err != nil { Warnf("JSON encode failed: %v\n", err) @@ -217,7 +220,7 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { return json.Marshal(outNode) } -func (p *ncduLsPrinter) Node(path string, node *restic.Node) { +func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) { out, err := lsNcduNode(path, node) if err != nil { Warnf("JSON encode failed: %v\n", err) @@ -249,8 +252,10 @@ type textLsPrinter struct { func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) { Verbosef("%v filtered by %v:\n", sn, p.dirs) } -func (p *textLsPrinter) Node(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) +func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) { + if !isPrefixDirectory { + Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) + } } func (p *textLsPrinter) LeaveDir(_ string) {} @@ -369,8 +374,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri printedDir := false if withinDir(nodepath) { - // if we're within a dir, print the node - printer.Node(nodepath, node) + // if we're within a target path, print the node + printer.Node(nodepath, node, false) printedDir = true // if recursive listing is requested, signal the walker that it @@ -383,12 +388,17 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri // if there's an upcoming match deeper in the tree (but we're not // there yet), signal the walker to descend into any subdirs if approachingMatchingTree(nodepath) { + // print node leading up to the target paths + if !printedDir { + printer.Node(nodepath, node, true) + } return nil } // otherwise, signal the walker to not walk recursively into any // subdirs if node.Type == "dir" { + // immediately generate leaveDir if the directory is skipped if printedDir { printer.LeaveDir(nodepath) } @@ -401,7 +411,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri ProcessNode: processNode, LeaveDir: func(path string) { // the root path `/` has no corresponding node and is thus also skipped by processNode - if withinDir(path) && path != "/" { + if path != "/" { printer.LeaveDir(path) } }, diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index 1b3c964e4..2b742d1b2 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -35,13 +35,16 @@ func TestRunLsNcdu(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - testRunInit(t, env.gopts) + testSetupBackupData(t, env) opts := BackupOptions{} testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) - ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"}) - assertIsValidJSON(t, ncdu) - - ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"}) - assertIsValidJSON(t, ncdu) + for _, paths := range [][]string{ + {"latest"}, + {"latest", "/testdata"}, + {"latest", "/testdata/0", "/testdata/0/tests"}, + } { + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, paths) + assertIsValidJSON(t, ncdu) + } } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 828b2920e..a59b5f81b 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -140,12 +140,12 @@ func TestLsNcdu(t *testing.T) { printer.Node("/directory", &restic.Node{ Type: "dir", Name: "directory", - }) + }, false) printer.Node("/directory/data", &restic.Node{ Type: "file", Name: "data", Size: 42, - }) + }, false) printer.LeaveDir("/directory") printer.Close() From 79c9fc45a9ce3fb98131b6a70d8521744bd75b5f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jul 2024 13:33:25 +0200 Subject: [PATCH 3/4] ls: fix disk size in --ncdu output Ncdu expects a size in disk blocks. Round up to the next full block. Otherwise, small files with size below 255bytes would be rounded down to 0B by ncdu. --- cmd/restic/cmd_ls.go | 9 ++++++--- cmd/restic/cmd_ls_test.go | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 8dd2af2d6..6e48b010f 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -193,10 +193,13 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { Mtime int64 `json:"mtime"` } + const blockSize = 512 + outNode := NcduNode{ - Name: node.Name, - Asize: node.Size, - Dsize: node.Size, + Name: node.Name, + Asize: node.Size, + // round up to nearest full blocksize + Dsize: (node.Size + blockSize - 1) / blockSize * blockSize, Dev: node.DeviceID, Ino: node.Inode, NLink: node.Links, diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index a59b5f81b..194975053 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -109,7 +109,7 @@ func TestLsNodeJSON(t *testing.T) { func TestLsNcduNode(t *testing.T) { for i, expect := range []string{ - `{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`, + `{"name":"baz","asize":12345,"dsize":12800,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`, `{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`, `{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`, `{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`, @@ -152,7 +152,7 @@ func TestLsNcdu(t *testing.T) { rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, [ {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}, - {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800} + {"name":"data","asize":42,"dsize":512,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800} ] ] `, buf.String()) From 5067a40bd829b44b853b4631e79c19f1f257d3f9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jul 2024 13:45:05 +0200 Subject: [PATCH 4/4] update ncdu changelog --- changelog/unreleased/issue-4549 | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/unreleased/issue-4549 b/changelog/unreleased/issue-4549 index 4829a9881..8f35b0233 100644 --- a/changelog/unreleased/issue-4549 +++ b/changelog/unreleased/issue-4549 @@ -9,3 +9,4 @@ You can use it as follows: `restic ls latest --ncdu | ncdu -f -` https://github.com/restic/restic/issues/4549 https://github.com/restic/restic/pull/4550 +https://github.com/restic/restic/pull/4911