diff --git a/bin/make_test_files.go b/bin/make_test_files.go deleted file mode 100644 index f2fd4b3f5..000000000 --- a/bin/make_test_files.go +++ /dev/null @@ -1,146 +0,0 @@ -// +build ignore - -// Build a directory structure with the required number of files in -// -// Run with go run make_test_files.go [flag] -package main - -import ( - cryptrand "crypto/rand" - "flag" - "io" - "log" - "math/rand" - "os" - "path/filepath" -) - -var ( - // Flags - numberOfFiles = flag.Int("n", 1000, "Number of files to create") - averageFilesPerDirectory = flag.Int("files-per-directory", 10, "Average number of files per directory") - maxDepth = flag.Int("max-depth", 10, "Maximum depth of directory hierarchy") - minFileSize = flag.Int64("min-size", 0, "Minimum size of file to create") - maxFileSize = flag.Int64("max-size", 100, "Maximum size of files to create") - minFileNameLength = flag.Int("min-name-length", 4, "Minimum size of file to create") - maxFileNameLength = flag.Int("max-name-length", 12, "Maximum size of files to create") - - directoriesToCreate int - totalDirectories int - fileNames = map[string]struct{}{} // keep a note of which file name we've used already -) - -// randomString create a random string for test purposes -func randomString(n int) string { - const ( - vowel = "aeiou" - consonant = "bcdfghjklmnpqrstvwxyz" - digit = "0123456789" - ) - pattern := []string{consonant, vowel, consonant, vowel, consonant, vowel, consonant, digit} - out := make([]byte, n) - p := 0 - for i := range out { - source := pattern[p] - p = (p + 1) % len(pattern) - out[i] = source[rand.Intn(len(source))] - } - return string(out) -} - -// fileName creates a unique random file or directory name -func fileName() (name string) { - for { - length := rand.Intn(*maxFileNameLength-*minFileNameLength) + *minFileNameLength - name = randomString(length) - if _, found := fileNames[name]; !found { - break - } - } - fileNames[name] = struct{}{} - return name -} - -// dir is a directory in the directory hierarchy being built up -type dir struct { - name string - depth int - children []*dir - parent *dir -} - -// Create a random directory hierarchy under d -func (d *dir) createDirectories() { - for totalDirectories < directoriesToCreate { - newDir := &dir{ - name: fileName(), - depth: d.depth + 1, - parent: d, - } - d.children = append(d.children, newDir) - totalDirectories++ - switch rand.Intn(4) { - case 0: - if d.depth < *maxDepth { - newDir.createDirectories() - } - case 1: - return - } - } - return -} - -// list the directory hierarchy -func (d *dir) list(path string, output []string) []string { - dirPath := filepath.Join(path, d.name) - output = append(output, dirPath) - for _, subDir := range d.children { - output = subDir.list(dirPath, output) - } - return output -} - -// writeFile writes a random file at dir/name -func writeFile(dir, name string) { - err := os.MkdirAll(dir, 0777) - if err != nil { - log.Fatalf("Failed to make directory %q: %v", dir, err) - } - path := filepath.Join(dir, name) - fd, err := os.Create(path) - if err != nil { - log.Fatalf("Failed to open file %q: %v", path, err) - } - size := rand.Int63n(*maxFileSize-*minFileSize) + *minFileSize - _, err = io.CopyN(fd, cryptrand.Reader, size) - if err != nil { - log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err) - } - err = fd.Close() - if err != nil { - log.Fatalf("Failed to close file %q: %v", path, err) - } -} - -func main() { - flag.Parse() - args := flag.Args() - if len(args) != 1 { - log.Fatalf("Require 1 directory argument") - } - outputDirectory := args[0] - log.Printf("Output dir %q", outputDirectory) - - directoriesToCreate = *numberOfFiles / *averageFilesPerDirectory - log.Printf("directoriesToCreate %v", directoriesToCreate) - root := &dir{name: outputDirectory, depth: 1} - for totalDirectories < directoriesToCreate { - root.createDirectories() - } - dirs := root.list("", []string{}) - for i := 0; i < *numberOfFiles; i++ { - dir := dirs[rand.Intn(len(dirs))] - writeFile(dir, fileName()) - } -} diff --git a/cmd/all/all.go b/cmd/all/all.go index 6e195eff6..6fae78757 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -54,6 +54,7 @@ import ( _ "github.com/rclone/rclone/cmd/sync" _ "github.com/rclone/rclone/cmd/test" _ "github.com/rclone/rclone/cmd/test/info" + _ "github.com/rclone/rclone/cmd/test/makefiles" _ "github.com/rclone/rclone/cmd/test/memory" _ "github.com/rclone/rclone/cmd/touch" _ "github.com/rclone/rclone/cmd/tree" diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go new file mode 100644 index 000000000..07db517f2 --- /dev/null +++ b/cmd/test/makefiles/makefiles.go @@ -0,0 +1,144 @@ +// Package makefiles builds a directory structure with the required +// number of files in of the required size. +package makefiles + +import ( + cryptrand "crypto/rand" + "io" + "log" + "math/rand" + "os" + "path/filepath" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/cmd/test" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/lib/random" + "github.com/spf13/cobra" +) + +var ( + // Flags + numberOfFiles = 1000 + averageFilesPerDirectory = 10 + maxDepth = 10 + minFileSize = fs.SizeSuffix(0) + maxFileSize = fs.SizeSuffix(100) + minFileNameLength = 4 + maxFileNameLength = 12 + + // Globals + directoriesToCreate int + totalDirectories int + fileNames = map[string]struct{}{} // keep a note of which file name we've used already +) + +func init() { + test.Command.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.IntVarP(cmdFlags, &numberOfFiles, "files", "", numberOfFiles, "Number of files to create") + flags.IntVarP(cmdFlags, &averageFilesPerDirectory, "files-per-directory", "", averageFilesPerDirectory, "Average number of files per directory") + flags.IntVarP(cmdFlags, &maxDepth, "max-depth", "", maxDepth, "Maximum depth of directory hierarchy") + flags.FVarP(cmdFlags, &minFileSize, "min-file-size", "", "Minimum size of file to create") + flags.FVarP(cmdFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create") + flags.IntVarP(cmdFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names") + flags.IntVarP(cmdFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names") +} + +var commandDefinition = &cobra.Command{ + Use: "makefiles ", + Short: `Make a random file hierarchy in `, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1, command, args) + outputDirectory := args[0] + directoriesToCreate = numberOfFiles / averageFilesPerDirectory + averageSize := (minFileSize + maxFileSize) / 2 + log.Printf("Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory) + root := &dir{name: outputDirectory, depth: 1} + for totalDirectories < directoriesToCreate { + root.createDirectories() + } + dirs := root.list("", []string{}) + for i := 0; i < numberOfFiles; i++ { + dir := dirs[rand.Intn(len(dirs))] + writeFile(dir, fileName()) + } + log.Printf("Done.") + }, +} + +// fileName creates a unique random file or directory name +func fileName() (name string) { + for { + length := rand.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength + name = random.String(length) + if _, found := fileNames[name]; !found { + break + } + } + fileNames[name] = struct{}{} + return name +} + +// dir is a directory in the directory hierarchy being built up +type dir struct { + name string + depth int + children []*dir + parent *dir +} + +// Create a random directory hierarchy under d +func (d *dir) createDirectories() { + for totalDirectories < directoriesToCreate { + newDir := &dir{ + name: fileName(), + depth: d.depth + 1, + parent: d, + } + d.children = append(d.children, newDir) + totalDirectories++ + switch rand.Intn(4) { + case 0: + if d.depth < maxDepth { + newDir.createDirectories() + } + case 1: + return + } + } + return +} + +// list the directory hierarchy +func (d *dir) list(path string, output []string) []string { + dirPath := filepath.Join(path, d.name) + output = append(output, dirPath) + for _, subDir := range d.children { + output = subDir.list(dirPath, output) + } + return output +} + +// writeFile writes a random file at dir/name +func writeFile(dir, name string) { + err := os.MkdirAll(dir, 0777) + if err != nil { + log.Fatalf("Failed to make directory %q: %v", dir, err) + } + path := filepath.Join(dir, name) + fd, err := os.Create(path) + if err != nil { + log.Fatalf("Failed to open file %q: %v", path, err) + } + size := rand.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize) + _, err = io.CopyN(fd, cryptrand.Reader, size) + if err != nil { + log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err) + } + err = fd.Close() + if err != nil { + log.Fatalf("Failed to close file %q: %v", path, err) + } +}