From c34f11a92faeb50c15ce3949269af66c5466e69a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Sat, 1 Apr 2017 20:53:44 +0200
Subject: [PATCH] rclone ncdu for exploring a remote with a text based user
interface.
---
cmd/all/all.go | 1 +
cmd/ncdu/ncdu.go | 542 ++++++++++++++++++++++++++++++++++++++++++
cmd/ncdu/scan/scan.go | 167 +++++++++++++
3 files changed, 710 insertions(+)
create mode 100644 cmd/ncdu/ncdu.go
create mode 100644 cmd/ncdu/scan/scan.go
diff --git a/cmd/all/all.go b/cmd/all/all.go
index c1db5a860..bc9d546ab 100644
--- a/cmd/all/all.go
+++ b/cmd/all/all.go
@@ -30,6 +30,7 @@ import (
_ "github.com/ncw/rclone/cmd/mount"
_ "github.com/ncw/rclone/cmd/move"
_ "github.com/ncw/rclone/cmd/moveto"
+ _ "github.com/ncw/rclone/cmd/ncdu"
_ "github.com/ncw/rclone/cmd/obscure"
_ "github.com/ncw/rclone/cmd/purge"
_ "github.com/ncw/rclone/cmd/rmdir"
diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
new file mode 100644
index 000000000..180c674b3
--- /dev/null
+++ b/cmd/ncdu/ncdu.go
@@ -0,0 +1,542 @@
+package ncdu
+
+import (
+ "fmt"
+ "log"
+ "path"
+ "sort"
+ "strings"
+
+ "github.com/ncw/rclone/cmd"
+ "github.com/ncw/rclone/cmd/ncdu/scan"
+ "github.com/ncw/rclone/fs"
+ termbox "github.com/nsf/termbox-go"
+ "github.com/pkg/errors"
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+ Use: "ncdu remote:path",
+ Short: `Explore a remote with a text based user interface.`,
+ Long: `
+This displays a text based user interface allowing the navigation of a
+remote. It is most useful for answering the question - "What is using
+all my disk space?".
+
+To make the user interface it first scans the entire remote given and
+builds an in memory representation. rclone ncdu can be used during
+this scanning phase and you will see it building up the directory
+structure as it goes along.
+
+Here are the keys - press '?' to toggle the help on and off
+
+ ` + strings.Join(helpText[1:], "\n ") + `
+
+This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
+rclone remotes. It is missing lots of features at the moment, most
+importantly deleting files, but is useful as it stands.
+`,
+ Run: func(command *cobra.Command, args []string) {
+ cmd.CheckArgs(1, 1, command, args)
+ fsrc := cmd.NewFsSrc(args)
+ cmd.Run(false, false, command, func() error {
+ return NewUI(fsrc).Show()
+ })
+ },
+}
+
+// help text
+var helpText = []string{
+ "rclone ncdu",
+ " ↑,↓ or k,j to Move",
+ " →,l to enter",
+ " ←,h to return",
+ " c toggle counts",
+ " g toggle graph",
+ " n,s,C sort by name,size,count",
+ " ? to toggle help on and off",
+ " q/ESC/c-C to quit",
+}
+
+// UI contains the state of the user interface
+type UI struct {
+ f fs.Fs // fs being displayed
+ fsName string // human name of Fs
+ root *scan.Dir // root directory
+ d *scan.Dir // current directory being displayed
+ path string // path of current directory
+ showBox bool // whether to show a box
+ boxText []string // text to show in box
+ entries fs.DirEntries // entries of current directory
+ sortPerm []int // order to display entries in after sorting
+ invSortPerm []int // inverse order
+ dirListHeight int // height of listing
+ listing bool // whether listing is in progress
+ showGraph bool // toggle showing graph
+ showCounts bool // toggle showing counts
+ sortByName int8 // +1 for normal, 0 for off, -1 for reverse
+ sortBySize int8
+ sortByCount int8
+ dirPosMap map[string]dirPos // store for directory positions
+}
+
+// Where we have got to in the directory listing
+type dirPos struct {
+ entry int
+ offset int
+}
+
+// Print a string
+func Print(x, y int, fg, bg termbox.Attribute, msg string) {
+ for _, c := range msg {
+ termbox.SetCell(x, y, c, fg, bg)
+ x++
+ }
+}
+
+// Printf a string
+func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface{}) {
+ s := fmt.Sprintf(format, args...)
+ Print(x, y, fg, bg, s)
+}
+
+// Line prints a string to given xmax, with given space
+func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
+ for _, c := range msg {
+ termbox.SetCell(x, y, c, fg, bg)
+ x++
+ if x >= xmax {
+ return
+ }
+ }
+ for ; x < xmax; x++ {
+ termbox.SetCell(x, y, spacer, fg, bg)
+ }
+}
+
+// Linef a string
+func Linef(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, format string, args ...interface{}) {
+ s := fmt.Sprintf(format, args...)
+ Line(x, y, xmax, fg, bg, spacer, s)
+}
+
+// Box the u.boxText onto the screen
+func (u *UI) Box() {
+ w, h := termbox.Size()
+
+ // Find dimensions of text
+ boxWidth := 10
+ for _, s := range u.boxText {
+ if len(s) > boxWidth && len(s) < w-4 {
+ boxWidth = len(s)
+ }
+ }
+ boxHeight := len(u.boxText)
+
+ // position
+ x := (w - boxWidth) / 2
+ y := (h - boxHeight) / 2
+ xmax := x + boxWidth
+
+ // draw text
+ fg, bg := termbox.ColorRed, termbox.ColorWhite
+ for i, s := range u.boxText {
+ Line(x, y+i, xmax, fg, bg, ' ', s)
+ fg = termbox.ColorBlack
+ }
+
+ // FIXME draw a box around
+}
+
+// find the biggest entry in the current listing
+func (u *UI) biggestEntry() (biggest int64) {
+ if u.d == nil {
+ return
+ }
+ for i := range u.entries {
+ size, _, _, _ := u.d.AttrI(u.sortPerm[i])
+ if size > biggest {
+ biggest = size
+ }
+ }
+ return
+}
+
+// Draw the current screen
+func (u *UI) Draw() error {
+ w, h := termbox.Size()
+ u.dirListHeight = h - 3
+
+ // Plot
+ err := termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
+ if err != nil {
+ return errors.Wrap(err, "failed to clear screen")
+ }
+
+ // Header line
+ Linef(0, 0, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version)
+
+ // Directory line
+ Linef(0, 1, w, termbox.ColorWhite, termbox.ColorBlack, '-', "-- %s ", u.path)
+
+ // graphs
+ const (
+ graphBars = 10
+ graph = "########## "
+ )
+
+ // Directory listing
+ if u.d != nil {
+ y := 2
+ perBar := u.biggestEntry() / graphBars
+ if perBar == 0 {
+ perBar = 1
+ }
+ dirPos := u.dirPosMap[u.path]
+ for i, j := range u.sortPerm[dirPos.offset:] {
+ entry := u.entries[j]
+ n := i + dirPos.offset
+ if y >= h-1 {
+ break
+ }
+ fg := termbox.ColorWhite
+ bg := termbox.ColorBlack
+ if n == dirPos.entry {
+ fg, bg = bg, fg
+ }
+ size, count, isDir, readable := u.d.AttrI(u.sortPerm[n])
+ mark := ' '
+ if isDir {
+ mark = '/'
+ }
+ message := ""
+ if !readable {
+ message = " [not read yet]"
+ }
+ extras := ""
+ if u.showCounts {
+ if count > 0 {
+ extras += fmt.Sprintf("%8v ", fs.SizeSuffix(count))
+ } else {
+ extras += " "
+ }
+
+ }
+ if u.showGraph {
+ bars := (size + perBar/2 - 1) / perBar
+ // clip if necessary - only happens during startup
+ if bars > 10 {
+ bars = 10
+ } else if bars < 0 {
+ bars = 0
+ }
+ extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
+ }
+ Linef(0, y, w, fg, bg, ' ', "%8v %s%c%s%s", fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
+ y++
+ }
+ }
+
+ // Footer
+ if u.d == nil {
+ Line(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Waiting for root directory...")
+ } else {
+ message := ""
+ if u.listing {
+ message = " [listing in progress]"
+ }
+ size, count := u.d.Attr()
+ Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %v, Objects: %d%s", fs.SizeSuffix(size), count, message)
+ }
+
+ // Show the box on top if requred
+ if u.showBox {
+ u.Box()
+ }
+ err = termbox.Flush()
+ if err != nil {
+ return errors.Wrap(err, "failed to flush screen")
+ }
+ return nil
+}
+
+// Move the cursor this many spaces adjusting the viewport as necessary
+func (u *UI) move(d int) {
+ if u.d == nil {
+ return
+ }
+
+ absD := d
+ if d < 0 {
+ absD = -d
+ }
+
+ entries := len(u.entries)
+
+ // Fetch current dirPos
+ dirPos := u.dirPosMap[u.path]
+
+ dirPos.entry += d
+
+ // check entry in range
+ if dirPos.entry < 0 {
+ dirPos.entry = 0
+ } else if dirPos.entry >= entries {
+ dirPos.entry = entries - 1
+ }
+
+ // check cursor still on screen
+ p := dirPos.entry - dirPos.offset // where dirPos.entry appears on the screen
+ if p < 0 {
+ dirPos.offset -= absD
+ } else if p >= u.dirListHeight {
+ dirPos.offset += absD
+ }
+
+ // check dirPos.offset in bounds
+ if dirPos.offset < 0 {
+ dirPos.offset = 0
+ } else if dirPos.offset >= entries {
+ dirPos.offset = entries - 1
+ }
+
+ // write dirPos back for later
+ u.dirPosMap[u.path] = dirPos
+}
+
+// Sort by the configured sort method
+type ncduSort struct {
+ sortPerm []int
+ entries fs.DirEntries
+ d *scan.Dir
+ u *UI
+}
+
+// Less is part of sort.Interface.
+func (ds *ncduSort) Less(i, j int) bool {
+ isize, icount, _, _ := ds.d.AttrI(ds.sortPerm[i])
+ jsize, jcount, _, _ := ds.d.AttrI(ds.sortPerm[j])
+ iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
+ switch {
+ case ds.u.sortByName < 0:
+ return iname > jname
+ case ds.u.sortByName > 0:
+ break
+ case ds.u.sortBySize < 0:
+ if isize != jsize {
+ return isize < jsize
+ }
+ case ds.u.sortBySize > 0:
+ if isize != jsize {
+ return isize > jsize
+ }
+ case ds.u.sortByCount < 0:
+ if icount != jcount {
+ return icount < jcount
+ }
+ case ds.u.sortByCount > 0:
+ if icount != jcount {
+ return icount > jcount
+ }
+ }
+ // if everything equal, sort by name
+ return iname < jname
+}
+
+// Swap is part of sort.Interface.
+func (ds *ncduSort) Swap(i, j int) {
+ ds.sortPerm[i], ds.sortPerm[j] = ds.sortPerm[j], ds.sortPerm[i]
+}
+
+// Len is part of sort.Interface.
+func (ds *ncduSort) Len() int {
+ return len(ds.sortPerm)
+}
+
+// sort the permutation map of the current directory
+func (u *UI) sortCurrentDir() {
+ u.sortPerm = u.sortPerm[:0]
+ for i := range u.entries {
+ u.sortPerm = append(u.sortPerm, i)
+ }
+ data := ncduSort{
+ sortPerm: u.sortPerm,
+ entries: u.entries,
+ d: u.d,
+ u: u,
+ }
+ sort.Sort(&data)
+ if len(u.invSortPerm) < len(u.sortPerm) {
+ u.invSortPerm = make([]int, len(u.sortPerm))
+ }
+ for i, j := range u.sortPerm {
+ u.invSortPerm[j] = i
+ }
+}
+
+// setCurrentDir sets the current directory
+func (u *UI) setCurrentDir(d *scan.Dir) {
+ u.d = d
+ u.entries = d.Entries()
+ u.path = path.Join(u.fsName, d.Path())
+ u.sortCurrentDir()
+}
+
+// enters the current entry
+func (u *UI) enter() {
+ if u.d == nil {
+ return
+ }
+ dirPos := u.dirPosMap[u.path]
+ d, _ := u.d.GetDir(u.sortPerm[dirPos.entry])
+ if d == nil {
+ return
+ }
+ u.setCurrentDir(d)
+}
+
+// up goes up to the parent directory
+func (u *UI) up() {
+ if u.d == nil {
+ return
+ }
+ parent := u.d.Parent()
+ if parent != nil {
+ u.setCurrentDir(parent)
+ }
+}
+
+// popupBox shows a box with the text in
+func (u *UI) popupBox(text []string) {
+ u.boxText = text
+ u.showBox = true
+}
+
+// togglePopupBox shows a box with the text in
+func (u *UI) togglePopupBox(text []string) {
+ if u.showBox {
+ u.showBox = false
+ } else {
+ u.popupBox(text)
+ }
+}
+
+// toggle the sorting for the flag passed in
+func (u *UI) toggleSort(sortType *int8) {
+ old := *sortType
+ u.sortBySize = 0
+ u.sortByCount = 0
+ u.sortByName = 0
+ if old == 0 {
+ *sortType = 1
+ } else {
+ *sortType = -old
+ }
+ u.sortCurrentDir()
+}
+
+// NewUI creates a new user interface for ncdu on f
+func NewUI(f fs.Fs) *UI {
+ return &UI{
+ f: f,
+ path: "Waiting for root...",
+ dirListHeight: 20, // updated in Draw
+ fsName: f.Name() + ":" + f.Root(),
+ showGraph: true,
+ showCounts: false,
+ sortByName: 0, // +1 for normal, 0 for off, -1 for reverse
+ sortBySize: 1,
+ sortByCount: 0,
+ dirPosMap: make(map[string]dirPos),
+ }
+}
+
+// Show shows the user interface
+func (u *UI) Show() error {
+ err := termbox.Init()
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer termbox.Close()
+
+ // scan the disk in the background
+ u.listing = true
+ rootChan, errChan, updated := scan.Scan(u.f)
+
+ // Poll the events into a channel
+ events := make(chan termbox.Event)
+ doneWithEvent := make(chan bool)
+ go func() {
+ for {
+ events <- termbox.PollEvent()
+ <-doneWithEvent
+ }
+ }()
+
+ // Main loop, waiting for events and channels
+outer:
+ for {
+ //Reset()
+ err := u.Draw()
+ if err != nil {
+ return errors.Wrap(err, "draw failed")
+ }
+ var root *scan.Dir
+ select {
+ case root = <-rootChan:
+ u.root = root
+ u.setCurrentDir(root)
+ case err := <-errChan:
+ if err != nil {
+ return errors.Wrap(err, "ncdu directory listing")
+ }
+ u.listing = false
+ case <-updated:
+ // redraw
+ // might want to limit updates per second
+ u.sortCurrentDir()
+ case ev := <-events:
+ doneWithEvent <- true
+ if ev.Type == termbox.EventKey {
+ switch ev.Key + termbox.Key(ev.Ch) {
+ case termbox.KeyEsc, termbox.KeyCtrlC, 'q':
+ if u.showBox {
+ u.showBox = false
+ } else {
+ break outer
+ }
+ case termbox.KeyArrowDown, 'j':
+ u.move(1)
+ case termbox.KeyArrowUp, 'k':
+ u.move(-1)
+ case termbox.KeyPgdn, '-', '_':
+ u.move(u.dirListHeight)
+ case termbox.KeyPgup, '=', '+':
+ u.move(-u.dirListHeight)
+ case termbox.KeyArrowLeft, 'h':
+ u.up()
+ case termbox.KeyArrowRight, 'l', termbox.KeyEnter:
+ u.enter()
+ case 'c':
+ u.showCounts = !u.showCounts
+ case 'g':
+ u.showGraph = !u.showGraph
+ case 'n':
+ u.toggleSort(&u.sortByName)
+ case 's':
+ u.toggleSort(&u.sortBySize)
+ case 'C':
+ u.toggleSort(&u.sortByCount)
+ case '?':
+ u.togglePopupBox(helpText)
+ }
+ }
+ }
+ // listen to key presses, etc
+ }
+ return nil
+}
diff --git a/cmd/ncdu/scan/scan.go b/cmd/ncdu/scan/scan.go
new file mode 100644
index 000000000..2449c8d9d
--- /dev/null
+++ b/cmd/ncdu/scan/scan.go
@@ -0,0 +1,167 @@
+// Package scan does concurrent scanning of an Fs building up a directory tree.
+package scan
+
+import (
+ "path"
+ "sync"
+
+ "github.com/ncw/rclone/fs"
+ "github.com/pkg/errors"
+)
+
+// Dir represents a directory found in the remote
+type Dir struct {
+ parent *Dir
+ path string
+ mu sync.Mutex
+ count int64
+ size int64
+ complete bool
+ entries fs.DirEntries
+ dirs map[string]*Dir
+ offset int // current listing offset
+ entry int // current listing entry
+}
+
+// Parent returns the directory above this one
+func (d *Dir) Parent() *Dir {
+ // no locking needed since these are write once in newDir()
+ return d.parent
+}
+
+// Path returns the position of the dir in the filesystem
+func (d *Dir) Path() string {
+ // no locking needed since these are write once in newDir()
+ return d.path
+}
+
+// make a new directory
+func newDir(parent *Dir, dirPath string, entries fs.DirEntries) *Dir {
+ d := &Dir{
+ parent: parent,
+ path: dirPath,
+ entries: entries,
+ dirs: make(map[string]*Dir),
+ }
+ // Count size in this dir
+ for _, entry := range entries {
+ if o, ok := entry.(fs.Object); ok {
+ d.count++
+ d.size += o.Size()
+ }
+ }
+ // Set my directory entry in parent
+ if parent != nil {
+ parent.mu.Lock()
+ leaf := path.Base(dirPath)
+ d.parent.dirs[leaf] = d
+ parent.mu.Unlock()
+ }
+ // Accumulate counts in parents
+ for ; parent != nil; parent = parent.parent {
+ parent.mu.Lock()
+ parent.count += d.count
+ parent.size += d.size
+ parent.mu.Unlock()
+ }
+ return d
+}
+
+// Entries returns a copy of the entries in the directory
+func (d *Dir) Entries() fs.DirEntries {
+ return append(fs.DirEntries(nil), d.entries...)
+}
+
+// gets the directory of the i-th entry
+//
+// returns nil if it is a file
+// returns a flag as to whether is directory or not
+//
+// Call with d.mu held
+func (d *Dir) getDir(i int) (subDir *Dir, isDir bool) {
+ obj := d.entries[i]
+ dir, ok := obj.(*fs.Dir)
+ if !ok {
+ return nil, false
+ }
+ leaf := path.Base(dir.Remote())
+ subDir = d.dirs[leaf]
+ return subDir, true
+}
+
+// GetDir returns the Dir of the i-th entry
+//
+// returns nil if it is a file
+// returns a flag as to whether is directory or not
+func (d *Dir) GetDir(i int) (subDir *Dir, isDir bool) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return d.getDir(i)
+}
+
+// Attr returns the size and count for the directory
+func (d *Dir) Attr() (size int64, count int64) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return d.size, d.count
+}
+
+// AttrI returns the size, count and flags for the i-th directory entry
+func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ subDir, isDir := d.getDir(i)
+ if !isDir {
+ return d.entries[i].Size(), 0, false, true
+ }
+ if subDir == nil {
+ return 0, 0, true, false
+ }
+ size, count = subDir.Attr()
+ return size, count, true, true
+}
+
+// Scan the Fs passed in, returning a root directory channel and an
+// error channel
+func Scan(f fs.Fs) (chan *Dir, chan error, chan struct{}) {
+ root := make(chan *Dir, 1)
+ errChan := make(chan error, 1)
+ updated := make(chan struct{}, 1)
+ go func() {
+ parents := map[string]*Dir{}
+ err := fs.Walk(f, "", false, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
+ if err != nil {
+ return err // FIXME mark directory as errored instead of aborting
+ }
+ var parent *Dir
+ if dirPath != "" {
+ parentPath := path.Dir(dirPath)
+ if parentPath == "." {
+ parentPath = ""
+ }
+ var ok bool
+ parent, ok = parents[parentPath]
+ if !ok {
+ errChan <- errors.Errorf("couldn't find parent for %q", dirPath)
+ }
+ }
+ d := newDir(parent, dirPath, entries)
+ parents[dirPath] = d
+ if dirPath == "" {
+ root <- d
+ }
+ // Mark updated
+ select {
+ case updated <- struct{}{}:
+ default:
+ break
+ }
+ return nil
+ })
+ if err != nil {
+ errChan <- errors.Wrap(err, "ncdu listing failed")
+ }
+ errChan <- nil
+ }()
+ return root, errChan, updated
+}