rclone/rclone.go

392 lines
8.5 KiB
Go
Raw Normal View History

2013-06-27 19:00:01 +00:00
// Sync files and directories to and from local and remote object stores
2013-06-27 18:51:03 +00:00
//
// Nick Craig-Wood <nick@craig-wood.com>
package main
import (
"fmt"
"log"
"os"
"runtime"
"runtime/pprof"
"strings"
"time"
2014-03-27 16:55:29 +00:00
"github.com/ogier/pflag"
"github.com/ncw/rclone/fs"
2013-06-27 19:13:07 +00:00
// Active file systems
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/s3"
_ "github.com/ncw/rclone/swift"
)
// Globals
var (
// Flags
2014-03-27 16:55:29 +00:00
cpuprofile = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval to print stats (0 to disable)")
2014-04-24 16:59:05 +00:00
version = pflag.BoolP("version", "V", false, "Print the version number")
logFile = pflag.StringP("log-file", "", "", "Log everything to this file")
)
2012-12-03 23:58:17 +00:00
type Command struct {
Name string
Help string
ArgsHelp string
Run func(fdst, fsrc fs.Fs)
MinArgs int
MaxArgs int
NoStats bool
2012-12-03 23:58:17 +00:00
}
// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
if len(args) < cmd.MinArgs {
2012-12-03 23:58:17 +00:00
syntaxError()
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
2012-12-03 23:58:17 +00:00
os.Exit(1)
} else if len(args) > cmd.MaxArgs {
2012-12-03 23:58:17 +00:00
syntaxError()
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
2012-12-03 23:58:17 +00:00
os.Exit(1)
}
}
var Commands = []Command{
{
Name: "copy",
2015-02-28 13:08:18 +00:00
ArgsHelp: "source:path dest:path",
Help: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing first by modification time then by
2015-02-28 13:08:18 +00:00
size. Doesn't delete files from the destination.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Sync(fdst, fsrc, false)
if err != nil {
log.Fatalf("Failed to copy: %v", err)
}
},
MinArgs: 2,
MaxArgs: 2,
},
{
Name: "sync",
2015-02-28 13:08:18 +00:00
ArgsHelp: "source:path dest:path",
Help: `
Sync the source to the destination. Doesn't transfer
unchanged files, testing first by modification time then by
2015-02-28 13:08:18 +00:00
size. Deletes any files that exist in source that don't
exist in destination. Since this can cause data loss, test
2014-03-27 16:55:29 +00:00
first with the --dry-run flag.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Sync(fdst, fsrc, true)
if err != nil {
log.Fatalf("Failed to sync: %v", err)
}
},
MinArgs: 2,
MaxArgs: 2,
2012-12-03 23:58:17 +00:00
},
{
Name: "ls",
2015-02-28 13:08:18 +00:00
ArgsHelp: "[remote:path]",
Help: `
List all the objects in the the path with size and path.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.List(fdst, os.Stdout)
if err != nil {
log.Fatalf("Failed to list: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
2013-01-23 22:43:20 +00:00
},
{
Name: "lsd",
2015-02-28 13:08:18 +00:00
ArgsHelp: "[remote:path]",
Help: `
List all directories/containers/buckets in the the path.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.ListDir(fdst, os.Stdout)
if err != nil {
log.Fatalf("Failed to listdir: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
2012-12-03 23:58:17 +00:00
},
{
Name: "lsl",
2015-02-28 13:08:18 +00:00
ArgsHelp: "[remote:path]",
Help: `
List all the objects in the the path with modification time, size and path.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.ListLong(fdst, os.Stdout)
if err != nil {
log.Fatalf("Failed to list long: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
},
{
Name: "md5sum",
2015-02-28 13:08:18 +00:00
ArgsHelp: "[remote:path]",
Help: `
2015-02-28 13:08:18 +00:00
Produces an md5sum file for all the objects in the path. This
is in the same format as the standard md5sum tool produces.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Md5sum(fdst, os.Stdout)
if err != nil {
log.Fatalf("Failed to list: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
},
2012-12-03 23:58:17 +00:00
{
Name: "mkdir",
2015-02-28 13:08:18 +00:00
ArgsHelp: "remote:path",
Help: `
2012-12-31 17:31:19 +00:00
Make the path if it doesn't already exist`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Mkdir(fdst)
if err != nil {
log.Fatalf("Failed to mkdir: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
2012-12-03 23:58:17 +00:00
},
{
Name: "rmdir",
2015-02-28 13:08:18 +00:00
ArgsHelp: "remote:path",
Help: `
Remove the path. Note that you can't remove a path with
2012-12-31 17:31:19 +00:00
objects in it, use purge for that.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Rmdir(fdst)
if err != nil {
log.Fatalf("Failed to rmdir: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
2012-12-03 23:58:17 +00:00
},
{
Name: "purge",
2015-02-28 13:08:18 +00:00
ArgsHelp: "remote:path",
Help: `
2012-12-31 17:31:19 +00:00
Remove the path and all of its contents.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Purge(fdst)
if err != nil {
log.Fatalf("Failed to purge: %v", err)
}
},
MinArgs: 1,
MaxArgs: 1,
},
2012-12-31 17:31:19 +00:00
{
Name: "check",
2015-02-28 13:08:18 +00:00
ArgsHelp: "source:path dest:path",
Help: `
2012-12-31 17:31:19 +00:00
Checks the files in the source and destination match. It
compares sizes and MD5SUMs and prints a report of files which
don't match. It doesn't alter the source or destination.`,
Run: func(fdst, fsrc fs.Fs) {
err := fs.Check(fdst, fsrc)
if err != nil {
log.Fatalf("Failed to check: %v", err)
}
},
MinArgs: 2,
MaxArgs: 2,
2012-12-31 17:31:19 +00:00
},
{
Name: "config",
Help: `
Enter an interactive configuration session.`,
Run: func(fdst, fsrc fs.Fs) {
fs.EditConfig()
},
NoStats: true,
},
{
Name: "help",
Help: `
2012-12-31 17:31:19 +00:00
This help.`,
NoStats: true,
2012-12-31 17:31:19 +00:00
},
2012-12-03 23:58:17 +00:00
}
// syntaxError prints the syntax
func syntaxError() {
2014-04-24 16:59:05 +00:00
fmt.Fprintf(os.Stderr, `Sync files and directories to and from local and remote object stores - %s.
2012-12-03 23:58:17 +00:00
Syntax: [options] subcommand <parameters> <parameters...>
Subcommands:
`, fs.Version)
2012-12-03 23:58:17 +00:00
for i := range Commands {
cmd := &Commands[i]
fmt.Fprintf(os.Stderr, " %s %s\n", cmd.Name, cmd.ArgsHelp)
fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
2012-12-03 23:58:17 +00:00
}
fmt.Fprintf(os.Stderr, "Options:\n")
2014-03-27 16:55:29 +00:00
pflag.PrintDefaults()
2012-12-03 23:58:17 +00:00
fmt.Fprintf(os.Stderr, `
It is only necessary to use a unique prefix of the subcommand, eg 'up' for 'upload'.
`)
}
// Exit with the message
func fatal(message string, args ...interface{}) {
syntaxError()
fmt.Fprintf(os.Stderr, message, args...)
os.Exit(1)
}
// Parse the command line flags
func ParseFlags() {
2014-03-27 16:55:29 +00:00
pflag.Usage = syntaxError
pflag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
fs.LoadConfig()
2013-06-29 09:43:52 +00:00
// Setup profiling if desired
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
2013-06-27 19:13:07 +00:00
fs.Stats.Error()
log.Fatal(err)
}
2014-07-25 17:19:49 +00:00
err = pprof.StartCPUProfile(f)
if err != nil {
fs.Stats.Error()
log.Fatal(err)
}
defer pprof.StopCPUProfile()
}
}
// Parse the command from the command line
func ParseCommand() (*Command, []string) {
args := pflag.Args()
if len(args) < 1 {
fatal("No command supplied\n")
}
2012-12-03 23:58:17 +00:00
cmd := strings.ToLower(args[0])
args = args[1:]
2012-12-03 23:58:17 +00:00
// Find the command doing a prefix match
var command *Command
2012-12-03 23:58:17 +00:00
for i := range Commands {
trialCommand := &Commands[i]
2012-12-03 23:58:17 +00:00
// exact command name found - use that
if trialCommand.Name == cmd {
command = trialCommand
2012-12-03 23:58:17 +00:00
break
} else if strings.HasPrefix(trialCommand.Name, cmd) {
if command != nil {
2013-06-27 19:13:07 +00:00
fs.Stats.Error()
2012-12-03 23:58:17 +00:00
log.Fatalf("Not unique - matches multiple commands %q", cmd)
}
command = trialCommand
}
}
if command == nil {
2013-06-27 19:13:07 +00:00
fs.Stats.Error()
2012-12-03 23:58:17 +00:00
log.Fatalf("Unknown command %q", cmd)
}
if command.Run == nil {
syntaxError()
}
command.checkArgs(args)
return command, args
}
// Create a Fs from a name
func NewFs(remote string) fs.Fs {
f, err := fs.NewFs(remote)
if err != nil {
fs.Stats.Error()
log.Fatalf("Failed to create file system for %q: %v", remote, err)
}
return f
}
// Print the stats every statsInterval
func StartStats() {
if *statsInterval <= 0 {
return
}
go func() {
ch := time.Tick(*statsInterval)
for {
<-ch
fs.Stats.Log()
}
}()
}
func main() {
ParseFlags()
2014-04-24 16:59:05 +00:00
if *version {
fmt.Printf("rclone %s\n", fs.Version)
2014-04-24 16:59:05 +00:00
os.Exit(0)
}
command, args := ParseCommand()
// Log file output
if *logFile != "" {
f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
if err != nil {
log.Fatalf("Failed to open log file: %v", err)
}
f.Seek(0, os.SEEK_END)
log.SetOutput(f)
redirectStderr(f)
}
// Make source and destination fs
2013-06-27 19:13:07 +00:00
var fdst, fsrc fs.Fs
if len(args) >= 1 {
fdst = NewFs(args[0])
}
if len(args) >= 2 {
fsrc = fdst
fdst = NewFs(args[1])
}
fs.CalculateModifyWindow(fdst, fsrc)
if !command.NoStats {
StartStats()
}
// Run the actual command
if command.Run != nil {
command.Run(fdst, fsrc)
if !command.NoStats {
fmt.Fprintln(os.Stderr, fs.Stats)
}
if fs.Config.Verbose {
2015-02-27 15:04:18 +00:00
fs.Debug(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
}
2013-06-27 19:13:07 +00:00
if fs.Stats.Errored() {
os.Exit(1)
}
os.Exit(0)
2012-12-31 17:31:19 +00:00
} else {
}
}