Implement fs.ListDir / lsd command

fs.go

@@ -17,6 +17,9 @@ type Fs interface {
 	// List the Fs into a channel
 	List() FsObjectsChan
 
+	// List the Fs directories/buckets/containers into a channel
+	ListDir() FsDirChan
+
 	// Find the FsObject at remote.  Returns nil if can't be found
 	NewFsObject(remote string) FsObject
 
@@ -82,6 +85,17 @@ type FsObjectsChan chan FsObject
 // A slice of FsObjects
 type FsObjects []FsObject
 
+// A structure of directory/container/bucket lists
+type FsDir struct {
+	Name  string    // name of the directory
+	When  time.Time // modification or creation time - IsZero for unknown
+	Bytes int64     // size of directory and contents -1 for unknown
+	Count int64     // number of objects -1 for unknown
+}
+
+// A channel of FsDir objects
+type FsDirChan chan *FsDir
+
 // NewFs makes a new Fs object from the path
 //
 // FIXME make more generic

fs_drive.go

@@ -568,6 +568,35 @@ func (f *FsDrive) List() FsObjectsChan {
 	return out
 }
 
+// Walk the path returning a channel of FsObjects
+func (f *FsDrive) ListDir() FsDirChan {
+	out := make(FsDirChan, *checkers)
+	go func() {
+		defer close(out)
+		err := f.findRoot(false)
+		if err != nil {
+			stats.Error()
+			log.Printf("Couldn't find root: %s", err)
+		} else {
+			_, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool {
+				dir := &FsDir{
+					Name:  item.Title,
+					Bytes: -1,
+					Count: -1,
+				}
+				dir.When, _ = time.Parse(time.RFC3339, item.ModifiedDate)
+				out <- dir
+				return false
+			})
+			if err != nil {
+				stats.Error()
+				log.Printf("ListDir failed: %s", err)
+			}
+		}
+	}()
+	return out
+}
+
 // Put the FsObject into the container
 //
 // Copy the reader in to the new object which is returned

fs_local.go

@@ -105,6 +105,49 @@ func (f *FsLocal) List() FsObjectsChan {
 	return out
 }
 
+// Walk the path returning a channel of FsObjects
+func (f *FsLocal) ListDir() FsDirChan {
+	out := make(FsDirChan, *checkers)
+	go func() {
+		defer close(out)
+		items, err := ioutil.ReadDir(f.root)
+		if err != nil {
+			stats.Error()
+			log.Printf("Couldn't find read directory: %s", err)
+		} else {
+			for _, item := range items {
+				if item.IsDir() {
+					dir := &FsDir{
+						Name:  item.Name(),
+						When:  item.ModTime(),
+						Bytes: 0,
+						Count: 0,
+					}
+					// Go down the tree to count the files and directories
+					dirpath := path.Join(f.root, item.Name())
+					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
+						if err != nil {
+							stats.Error()
+							log.Printf("Failed to open directory: %s: %s", path, err)
+						} else {
+							dir.Count += 1
+							dir.Bytes += fi.Size()
+						}
+						return nil
+					})
+					if err != nil {
+						stats.Error()
+						log.Printf("Failed to open directory: %s: %s", dirpath, err)
+					}
+					out <- dir
+				}
+			}
+		}
+		// err := f.findRoot(false)
+	}()
+	return out
+}
+
 // Puts the FsObject to the local filesystem
 func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) {
 	dstPath := filepath.Join(f.root, remote)

fs_s3.go

@@ -1,6 +1,8 @@
 // S3 interface
 package main
 
+// FIXME need to prevent anything but ListDir working for s3://
+
 import (
 	"errors"
 	"flag"
@@ -65,7 +67,7 @@ func (f *FsS3) String() string {
 }
 
 // Pattern to match a s3 url
-var s3Match = regexp.MustCompile(`^s3://([^/]+)(.*)$`)
+var s3Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`)
 
 // parseParse parses a s3 'url'
 func s3ParsePath(path string) (bucket, directory string, err error) {
@@ -132,23 +134,6 @@ func NewFsS3(path string) (*FsS3, error) {
 	return f, nil
 }
 
-// Lists the buckets
-func S3Buckets() {
-	c, err := s3Connection()
-	if err != nil {
-		stats.Error()
-		log.Fatalf("Couldn't connect: %s", err)
-	}
-	buckets, err := c.List()
-	if err != nil {
-		stats.Error()
-		log.Fatalf("Couldn't list buckets: %s", err)
-	}
-	for _, bucket := range buckets.Buckets {
-		fmt.Printf("%12s %s\n", bucket.CreationDate, bucket.Name)
-	}
-}
-
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
@@ -206,6 +191,30 @@ func (f *FsS3) List() FsObjectsChan {
 	return out
 }
 
+// Lists the buckets
+func (f *FsS3) ListDir() FsDirChan {
+	out := make(FsDirChan, *checkers)
+	go func() {
+		defer close(out)
+		buckets, err := f.c.List()
+		if err != nil {
+			stats.Error()
+			log.Printf("Couldn't list buckets: %s", err)
+		} else {
+			for _, bucket := range buckets.Buckets {
+				when, _ := time.Parse(time.RFC3339, bucket.CreationDate)
+				out <- &FsDir{
+					Name:  bucket.Name,
+					When:  when,
+					Bytes: -1,
+					Count: -1,
+				}
+			}
+		}
+	}()
+	return out
+}
+
 // Put the FsObject into the bucket
 func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) {
 	// Temporary FsObject under construction

fs_swift.go

@@ -1,6 +1,8 @@
 // Swift interface
 package main
 
+// FIXME need to prevent anything but ListDir working for swift://
+
 import (
 	"errors"
 	"flag"
@@ -48,7 +50,7 @@ func (f *FsSwift) String() string {
 }
 
 // Pattern to match a swift url
-var swiftMatch = regexp.MustCompile(`^swift://([^/]+)(.*)$`)
+var swiftMatch = regexp.MustCompile(`^swift://([^/]*)(.*)$`)
 
 // parseParse parses a swift 'url'
 func parsePath(path string) (container, directory string, err error) {
@@ -102,23 +104,6 @@ func NewFsSwift(path string) (*FsSwift, error) {
 	return f, nil
 }
 
-// Lists the containers
-func SwiftContainers() {
-	c, err := swiftConnection()
-	if err != nil {
-		stats.Error()
-		log.Fatalf("Couldn't connect: %s", err)
-	}
-	containers, err := c.ContainersAll(nil)
-	if err != nil {
-		stats.Error()
-		log.Fatalf("Couldn't list containers: %s", err)
-	}
-	for _, container := range containers {
-		fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
-	}
-}
-
 // Return an FsObject from a path
 //
 // May return nil if an error occurred
@@ -173,6 +158,28 @@ func (f *FsSwift) List() FsObjectsChan {
 	return out
 }
 
+// Lists the containers
+func (f *FsSwift) ListDir() FsDirChan {
+	out := make(FsDirChan, *checkers)
+	go func() {
+		defer close(out)
+		containers, err := f.c.ContainersAll(nil)
+		if err != nil {
+			stats.Error()
+			log.Printf("Couldn't list containers: %s", err)
+		} else {
+			for _, container := range containers {
+				out <- &FsDir{
+					Name:  container.Name,
+					Bytes: container.Bytes,
+					Count: container.Count,
+				}
+			}
+		}
+	}()
+	return out
+}
+
 // Put the FsObject into the container
 //
 // Copy the reader in to the new object which is returned

notes.txt

@@ -1,3 +1,19 @@
+Names
+  * cakesync
+  * zzsync
+  * pqrsync
+  * gogosync
+  * pogosync
+  * oursync
+  * godupe
+  * gobsync
+  * gobcopy
+  * togosync
+  * tangosync
+  * mangosync
+  * cargosync
+  * cargocopy - domain available
+
 Todo
   * Make a test suite which can run on all the given types of fs
   * Copy should use the sync code as it is more efficient at directory listing

swiftsync.go

@@ -300,7 +300,7 @@ func Check(fdst, fsrc Fs) {
 // List the Fs to stdout
 //
 // Lists in parallel which may get them out of order
-func List(f Fs) {
+func List(f, _ Fs) {
 	in := f.List()
 	var wg sync.WaitGroup
 	wg.Add(*checkers)
@@ -318,14 +318,11 @@ func List(f Fs) {
 	wg.Wait()
 }
 
-// Lists files in a container
+// List the directories/buckets/containers in the Fs to stdout
-func list(fdst, fsrc Fs) {
+func ListDir(f, _ Fs) {
-	if fdst == nil {
+	for dir := range f.ListDir() {
-		// FIXMESwiftContainers()
+		fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
-		S3Buckets()
-		return
 	}
-	List(fdst)
 }
 
 // Makes a destination directory or container
@@ -417,11 +414,19 @@ var Commands = []Command{
 		"ls",
 		`[<path>]
 
-        List the path. If no parameter is supplied then it lists the
+        List all the objects in the the path.`,
-        available swift containers.`,
 
-		list,
+		List,
-		0, 1,
+		1, 1,
+	},
+	{
+		"lsd",
+		`[<path>]
+
+        List all directoryes/objects/buckets in the the path.`,
+
+		ListDir,
+		1, 1,
 	},
 	{
 		"mkdir",