Merge pull request #4911 from MichaelEischer/fix-filtered-ls-ncdu

ls: Fix --ncdu output with path filters and fix disk size

changelog/unreleased/issue-4549

@@ -9,3 +9,4 @@ You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
 
 https://github.com/restic/restic/issues/4549
 https://github.com/restic/restic/pull/4550
+https://github.com/restic/restic/pull/4911

cmd/restic/cmd_ls.go

@@ -71,7 +71,7 @@ func init() {
 
 type lsPrinter interface {
 	Snapshot(sn *restic.Snapshot)
-	Node(path string, node *restic.Node)
+	Node(path string, node *restic.Node, isPrefixDirectory bool)
 	LeaveDir(path string)
 	Close()
 }
@@ -102,7 +102,10 @@ func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
 }
 
 // Print node in our custom JSON format, followed by a newline.
-func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
+func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
+	if isPrefixDirectory {
+		return
+	}
 	err := lsNodeJSON(p.enc, path, node)
 	if err != nil {
 		Warnf("JSON encode failed: %v\n", err)
@@ -190,10 +193,13 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 		Mtime  int64  `json:"mtime"`
 	}
 
+	const blockSize = 512
+
 	outNode := NcduNode{
-		Name:   node.Name,
+		Name:  node.Name,
-		Asize:  node.Size,
+		Asize: node.Size,
-		Dsize:  node.Size,
+		// round up to nearest full blocksize
+		Dsize:  (node.Size + blockSize - 1) / blockSize * blockSize,
 		Dev:    node.DeviceID,
 		Ino:    node.Inode,
 		NLink:  node.Links,
@@ -217,7 +223,7 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 	return json.Marshal(outNode)
 }
 
-func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
+func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) {
 	out, err := lsNcduNode(path, node)
 	if err != nil {
 		Warnf("JSON encode failed: %v\n", err)
@@ -249,8 +255,10 @@ type textLsPrinter struct {
 func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
 	Verbosef("%v filtered by %v:\n", sn, p.dirs)
 }
-func (p *textLsPrinter) Node(path string, node *restic.Node) {
+func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
-	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+	if !isPrefixDirectory {
+		Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+	}
 }
 
 func (p *textLsPrinter) LeaveDir(_ string) {}
@@ -367,9 +375,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			return nil
 		}
 
+		printedDir := false
 		if withinDir(nodepath) {
-			// if we're within a dir, print the node
+			// if we're within a target path, print the node
-			printer.Node(nodepath, node)
+			printer.Node(nodepath, node, false)
+			printedDir = true
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
@@ -381,12 +391,20 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		// if there's an upcoming match deeper in the tree (but we're not
 		// there yet), signal the walker to descend into any subdirs
 		if approachingMatchingTree(nodepath) {
+			// print node leading up to the target paths
+			if !printedDir {
+				printer.Node(nodepath, node, true)
+			}
 			return nil
 		}
 
 		// otherwise, signal the walker to not walk recursively into any
 		// subdirs
 		if node.Type == "dir" {
+			// immediately generate leaveDir if the directory is skipped
+			if printedDir {
+				printer.LeaveDir(nodepath)
+			}
 			return walker.ErrSkipNode
 		}
 		return nil
@@ -396,7 +414,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		ProcessNode: processNode,
 		LeaveDir: func(path string) {
 			// the root path `/` has no corresponding node and is thus also skipped by processNode
-			if withinDir(path) && path != "/" {
+			if path != "/" {
 				printer.LeaveDir(path)
 			}
 		},

cmd/restic/cmd_ls_integration_test.go

@@ -35,13 +35,16 @@ func TestRunLsNcdu(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
-	testRunInit(t, env.gopts)
+	testSetupBackupData(t, env)
 	opts := BackupOptions{}
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 
-	ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
+	for _, paths := range [][]string{
-	assertIsValidJSON(t, ncdu)
+		{"latest"},
-
+		{"latest", "/testdata"},
-	ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
+		{"latest", "/testdata/0", "/testdata/0/tests"},
-	assertIsValidJSON(t, ncdu)
+	} {
+		ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, paths)
+		assertIsValidJSON(t, ncdu)
+	}
 }

cmd/restic/cmd_ls_test.go

@@ -109,7 +109,7 @@ func TestLsNodeJSON(t *testing.T) {
 
 func TestLsNcduNode(t *testing.T) {
 	for i, expect := range []string{
-		`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
+		`{"name":"baz","asize":12345,"dsize":12800,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
 		`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
 		`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
 		`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
@@ -140,19 +140,19 @@ func TestLsNcdu(t *testing.T) {
 	printer.Node("/directory", &restic.Node{
 		Type: "dir",
 		Name: "directory",
-	})
+	}, false)
 	printer.Node("/directory/data", &restic.Node{
 		Type: "file",
 		Name: "data",
 		Size: 42,
-	})
+	}, false)
 	printer.LeaveDir("/directory")
 	printer.Close()
 
 	rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
   [
     {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
-    {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
+    {"name":"data","asize":42,"dsize":512,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
   ]
 ]
 `, buf.String())