rclone/swiftsync.go
2012-11-28 23:40:09 +00:00

339 lines
8.7 KiB
Go

// Sync files and directories to and from swift
//
// Nick Craig-Wood <nick@craig-wood.com>
package main
import (
"crypto/md5"
"flag"
"fmt"
"github.com/ncw/swift"
"io"
"log"
"os"
"path/filepath"
"runtime/pprof"
"strings"
)
// Globals
var (
// Flags
//fileSize = flag.Int64("s", 1E9, "Size of the check files")
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
//duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
//logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
)
type FsObject struct {
rel string
path string
info os.FileInfo
}
type FsObjects []FsObject
// Write debuging output for this FsObject
func (fs *FsObject) Debugf(text string, args ...interface{}) {
out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.rel, out)
}
// md5sum calculates the md5sum of a file returning a lowercase hex string
func (fs *FsObject) md5sum() (string, error) {
in, err := os.Open(fs.path)
if err != nil {
fs.Debugf("Failed to open: %s", err)
return "", err
}
defer in.Close() // FIXME ignoring error
hash := md5.New()
_, err = io.Copy(hash, in)
if err != nil {
fs.Debugf("Failed to read: %s", err)
return "", err
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
// Checks to see if an object has changed or not by looking at its size, mtime and MD5SUM
//
// If the remote object size is different then it is considered to be
// changed.
//
// If the size is the same and the mtime is the same then it is
// considered to be unchanged. This is the heuristic rsync uses when
// not using --checksum.
//
// If the size is the same and and mtime is different or unreadable
// and the MD5SUM is the same then the file is considered to be
// unchanged. In this case the mtime on the object is updated.
//
// Otherwise the file is considered to be changed.
func (fs *FsObject) changed(c *swift.Connection, container string) bool {
obj, h, err := c.Object(container, fs.rel)
if err != nil {
fs.Debugf("Failed to read info: %s", err)
return true
}
if obj.Bytes != fs.info.Size() {
fs.Debugf("Sizes differ")
return true
}
// Size the same so check the mtime
m := h.ObjectMetadata()
t, err := m.GetModTime()
if err != nil {
fs.Debugf("Failed to read mtime: %s", err)
} else if !t.Equal(fs.info.ModTime()) {
fs.Debugf("Modification times differ")
} else {
fs.Debugf("Size and modification time the same - skipping")
return false
}
// mtime is unreadable or different but size is the same so
// check the MD5SUM
localMd5, err := fs.md5sum()
if err != nil {
fs.Debugf("Failed to calculate md5: %s", err)
return true
}
// fs.Debugf("Local MD5 %s", localMd5)
// fs.Debugf("Remote MD5 %s", obj.Hash)
if localMd5 != strings.ToLower(obj.Hash) {
fs.Debugf("Md5sums differ")
return true
}
// Update the mtime of the remote object here
m.SetModTime(fs.info.ModTime())
err = c.ObjectUpdate(container, fs.rel, m.ObjectHeaders())
if err != nil {
fs.Debugf("Failed to update mtime: %s", err)
}
fs.Debugf("Updated mtime")
fs.Debugf("Size and MD5SUM identical - skipping")
return false
}
// Puts the FsObject into the container
func (fs *FsObject) put(c *swift.Connection, container string) {
mode := fs.info.Mode()
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
fs.Debugf("Can't transfer non file/directory")
} else if mode&os.ModeDir != 0 {
// Debug?
fs.Debugf("FIXME Skipping directory")
} else {
// Check to see if changed or not
if !fs.changed(c, container) {
fs.Debugf("Unchanged skipping")
return
}
// FIXME content type
in, err := os.Open(fs.path)
if err != nil {
fs.Debugf("Failed to open: %s", err)
return
}
defer in.Close()
m := swift.Metadata{}
m.SetModTime(fs.info.ModTime())
_, err = c.ObjectPut(container, fs.rel, in, true, "", "", m.ObjectHeaders())
if err != nil {
fs.Debugf("Failed to upload: %s", err)
return
}
fs.Debugf("Uploaded")
}
}
// Walk the path
//
// FIXME ignore symlinks?
// FIXME what about hardlinks / etc
func walk(root string) FsObjects {
files := make(FsObjects, 0, 1024)
err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
if err != nil {
log.Printf("Failed to open directory: %s: %s", path, err)
} else {
info, err := os.Lstat(path)
if err != nil {
log.Printf("Failed to stat %s: %s", path, err)
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
log.Printf("Failed to get relative path %s: %s", path, err)
return nil
}
if rel == "." {
rel = ""
}
files = append(files, FsObject{rel: rel, path: path, info: info})
}
return nil
})
if err != nil {
log.Printf("Failed to open directory: %s: %s", root, err)
}
return files
}
// syntaxError prints the syntax
func syntaxError() {
fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
FIXME
Full options:
`)
flag.PrintDefaults()
}
// Exit with the message
func fatal(message string, args ...interface{}) {
syntaxError()
fmt.Fprintf(os.Stderr, message, args...)
os.Exit(1)
}
// checkArgs checks there are enough arguments and prints a message if not
func checkArgs(args []string, n int, message string) {
if len(args) != n {
syntaxError()
fmt.Fprintf(os.Stderr, "%d arguments required: %s\n", n, message)
os.Exit(1)
}
}
// uploads a file into a container
func upload(c *swift.Connection, root, container string) {
files := walk(root)
for _, fs := range files {
fs.put(c, container)
}
}
// Lists the containers
func listContainers(c *swift.Connection) {
containers, err := c.ContainersAll(nil)
if err != nil {
log.Fatalf("Couldn't list containers: %s", err)
}
for _, container := range containers {
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
}
}
// Lists files in a container
func list(c *swift.Connection, container string) {
//objects, err := c.ObjectsAll(container, &swift.ObjectsOpts{Prefix: "", Delimiter: '/'})
objects, err := c.ObjectsAll(container, nil)
if err != nil {
log.Fatalf("Couldn't read container %q: %s", container, err)
}
for _, object := range objects {
if object.PseudoDirectory {
fmt.Printf("%9s %19s %s\n", "Directory", "-", object.Name)
} else {
fmt.Printf("%9d %19s %s\n", object.Bytes, object.LastModified.Format("2006-01-02 15:04:05"), object.Name)
}
}
}
// Makes a container
func mkdir(c *swift.Connection, container string) {
err := c.ContainerCreate(container, nil)
if err != nil {
log.Fatalf("Couldn't create container %q: %s", container, err)
}
}
// Removes a container
func rmdir(c *swift.Connection, container string) {
err := c.ContainerDelete(container)
if err != nil {
log.Fatalf("Couldn't delete container %q: %s", container, err)
}
}
func main() {
flag.Usage = syntaxError
flag.Parse()
args := flag.Args()
//runtime.GOMAXPROCS(3)
// Setup profiling if desired
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
fmt.Println(args)
if len(args) < 1 {
fatal("No command supplied\n")
}
if *userName == "" {
log.Fatal("Need -user or environmental variable ST_USER")
}
if *apiKey == "" {
log.Fatal("Need -key or environmental variable ST_KEY")
}
if *authUrl == "" {
log.Fatal("Need -auth or environmental variable ST_AUTH")
}
c := &swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
}
err := c.Authenticate()
if err != nil {
log.Fatal("Failed to authenticate", err)
}
command := args[0]
args = args[1:]
switch command {
case "up", "upload":
checkArgs(args, 2, "Need directory to read from and container to write to")
upload(c, args[0], args[1])
case "list", "ls":
if len(args) == 0 {
listContainers(c)
} else {
checkArgs(args, 1, "Need container to list")
list(c, args[0])
}
case "mkdir":
checkArgs(args, 1, "Need container to make")
mkdir(c, args[0])
case "rmdir":
checkArgs(args, 1, "Need container to delte")
rmdir(c, args[0])
default:
log.Fatalf("Unknown command %q", command)
}
}