2012-12-26 12:23:58 +00:00
|
|
|
// File system interface
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
package fs
|
2012-12-26 12:23:58 +00:00
|
|
|
|
|
|
|
import (
|
2012-12-31 16:40:34 +00:00
|
|
|
"fmt"
|
2012-12-26 12:23:58 +00:00
|
|
|
"io"
|
2012-12-31 16:40:34 +00:00
|
|
|
"log"
|
2013-06-27 19:13:07 +00:00
|
|
|
"regexp"
|
2012-12-26 12:23:58 +00:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Filesystem registry
|
2014-03-15 16:06:11 +00:00
|
|
|
fsRegistry []*FsInfo
|
2013-06-27 19:13:07 +00:00
|
|
|
)
|
|
|
|
|
2014-03-15 16:06:11 +00:00
|
|
|
// Filesystem info
|
|
|
|
type FsInfo struct {
|
|
|
|
Name string // name of this fs
|
|
|
|
NewFs func(string, string) (Fs, error) // create a new file system
|
2014-03-16 13:54:43 +00:00
|
|
|
Config func(string) // function to call to help with config
|
2014-03-15 16:06:11 +00:00
|
|
|
Options []Option
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2014-03-15 16:06:11 +00:00
|
|
|
// An options for a Fs
|
|
|
|
type Option struct {
|
|
|
|
Name string
|
|
|
|
Help string
|
|
|
|
Optional bool
|
|
|
|
Examples []OptionExample
|
|
|
|
}
|
|
|
|
|
|
|
|
// An example for an option
|
|
|
|
type OptionExample struct {
|
|
|
|
Value string
|
|
|
|
Help string
|
|
|
|
}
|
|
|
|
|
|
|
|
// Choose an option
|
|
|
|
func (o *Option) Choose() string {
|
|
|
|
fmt.Println(o.Help)
|
|
|
|
if len(o.Examples) > 0 {
|
|
|
|
var values []string
|
|
|
|
var help []string
|
|
|
|
for _, example := range o.Examples {
|
|
|
|
values = append(values, example.Value)
|
|
|
|
help = append(help, example.Help)
|
|
|
|
}
|
|
|
|
return Choose(o.Name, values, help, true)
|
|
|
|
}
|
|
|
|
fmt.Printf("%s> ", o.Name)
|
|
|
|
return ReadLine()
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Register a filesystem
|
|
|
|
//
|
|
|
|
// Fs modules should use this in an init() function
|
2014-03-15 16:06:11 +00:00
|
|
|
func Register(info *FsInfo) {
|
|
|
|
fsRegistry = append(fsRegistry, info)
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2012-12-26 12:23:58 +00:00
|
|
|
// A Filesystem, describes the local filesystem and the remote object store
|
|
|
|
type Fs interface {
|
2013-01-18 18:54:19 +00:00
|
|
|
// String returns a description of the FS
|
2012-12-31 16:40:34 +00:00
|
|
|
String() string
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// List the Fs into a channel
|
2013-06-28 07:57:32 +00:00
|
|
|
List() ObjectsChan
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2013-01-23 22:43:20 +00:00
|
|
|
// List the Fs directories/buckets/containers into a channel
|
2013-06-28 07:57:32 +00:00
|
|
|
ListDir() DirChan
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// Find the Object at remote. Returns nil if can't be found
|
|
|
|
NewFsObject(remote string) Object
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2013-06-28 07:57:32 +00:00
|
|
|
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Make the directory (container, bucket)
|
2012-12-26 12:23:58 +00:00
|
|
|
Mkdir() error
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Remove the directory (container, bucket) if empty
|
2012-12-26 12:23:58 +00:00
|
|
|
Rmdir() error
|
2013-01-18 23:21:02 +00:00
|
|
|
|
|
|
|
// Precision of the ModTimes in this Fs
|
|
|
|
Precision() time.Duration
|
2012-12-26 12:23:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// FIXME make f.Debugf...
|
|
|
|
|
|
|
|
// A filesystem like object which can either be a remote object or a
|
|
|
|
// local file/directory
|
2013-06-28 07:57:32 +00:00
|
|
|
type Object interface {
|
2013-01-18 18:54:19 +00:00
|
|
|
// Remote returns the remote path
|
2012-12-26 12:23:58 +00:00
|
|
|
Remote() string
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Md5sum returns the md5 checksum of the file
|
2012-12-26 12:23:58 +00:00
|
|
|
Md5sum() (string, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// ModTime returns the modification date of the file
|
2013-01-02 15:21:55 +00:00
|
|
|
ModTime() time.Time
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// SetModTime sets the metadata on the object to set the modification date
|
2012-12-26 12:23:58 +00:00
|
|
|
SetModTime(time.Time)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Size returns the size of the file
|
2012-12-26 12:23:58 +00:00
|
|
|
Size() int64
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
2012-12-26 12:23:58 +00:00
|
|
|
Open() (io.ReadCloser, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Storable says whether this object can be stored
|
2012-12-26 12:23:58 +00:00
|
|
|
Storable() bool
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Removes this object
|
2012-12-26 12:23:58 +00:00
|
|
|
Remove() error
|
|
|
|
}
|
|
|
|
|
2013-01-18 17:01:47 +00:00
|
|
|
// Optional interfaces
|
|
|
|
type Purger interface {
|
|
|
|
// Purge all files in the root and the root directory
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
|
|
|
Purge() error
|
|
|
|
}
|
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// A channel of Objects
|
|
|
|
type ObjectsChan chan Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// A slice of Objects
|
|
|
|
type Objects []Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2013-01-23 22:43:20 +00:00
|
|
|
// A structure of directory/container/bucket lists
|
2013-06-28 07:57:32 +00:00
|
|
|
type Dir struct {
|
2013-01-23 22:43:20 +00:00
|
|
|
Name string // name of the directory
|
|
|
|
When time.Time // modification or creation time - IsZero for unknown
|
|
|
|
Bytes int64 // size of directory and contents -1 for unknown
|
|
|
|
Count int64 // number of objects -1 for unknown
|
|
|
|
}
|
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// A channel of Dir objects
|
|
|
|
type DirChan chan *Dir
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2014-03-15 16:06:11 +00:00
|
|
|
// Pattern to match a url
|
2014-03-27 17:31:57 +00:00
|
|
|
var matcher = regexp.MustCompile(`^([\w_-]+):(.*)$`)
|
2014-03-15 16:06:11 +00:00
|
|
|
|
|
|
|
// Finds a FsInfo object for the name passed in
|
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
|
|
|
func Find(name string) (*FsInfo, error) {
|
|
|
|
for _, item := range fsRegistry {
|
|
|
|
if item.Name == name {
|
|
|
|
return item, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("Didn't find filing system for %q", name)
|
|
|
|
}
|
|
|
|
|
2012-12-29 11:35:41 +00:00
|
|
|
// NewFs makes a new Fs object from the path
|
|
|
|
//
|
2014-03-15 16:06:11 +00:00
|
|
|
// The path is of the form service://path
|
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
2012-12-29 11:35:41 +00:00
|
|
|
func NewFs(path string) (Fs, error) {
|
2014-03-15 16:06:11 +00:00
|
|
|
parts := matcher.FindStringSubmatch(path)
|
|
|
|
fsName, configName, fsPath := "local", "local", path
|
|
|
|
if parts != nil {
|
|
|
|
configName, fsPath = parts[1], parts[2]
|
|
|
|
var err error
|
|
|
|
fsName, err = ConfigFile.GetValue(configName, "type")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
2013-01-14 23:38:18 +00:00
|
|
|
}
|
2014-03-15 16:06:11 +00:00
|
|
|
fs, err := Find(fsName)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return fs.NewFs(configName, fsPath)
|
2012-12-29 11:35:41 +00:00
|
|
|
}
|
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// Write debuging output for this Object
|
|
|
|
func Debug(fs Object, text string, args ...interface{}) {
|
2013-06-27 19:13:07 +00:00
|
|
|
if Config.Verbose {
|
2012-12-31 16:40:34 +00:00
|
|
|
out := fmt.Sprintf(text, args...)
|
|
|
|
log.Printf("%s: %s", fs.Remote(), out)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
// Write log output for this Object
|
|
|
|
func Log(fs Object, text string, args ...interface{}) {
|
2013-06-27 19:13:07 +00:00
|
|
|
if !Config.Quiet {
|
2012-12-31 16:40:34 +00:00
|
|
|
out := fmt.Sprintf(text, args...)
|
|
|
|
log.Printf("%s: %s", fs.Remote(), out)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-12-26 12:23:58 +00:00
|
|
|
// checkClose is a utility function used to check the return from
|
|
|
|
// Close in a defer statement.
|
|
|
|
func checkClose(c io.Closer, err *error) {
|
|
|
|
cerr := c.Close()
|
|
|
|
if *err == nil {
|
|
|
|
*err = cerr
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-27 17:26:24 +00:00
|
|
|
// Work out modify window for fses passed in - sets Config.ModifyWindow
|
|
|
|
//
|
|
|
|
// This is the largest modify window of all the fses in use, and the
|
|
|
|
// user configured value
|
|
|
|
func CalculateModifyWindow(fs ...Fs) {
|
|
|
|
for _, f := range fs {
|
|
|
|
if f != nil {
|
|
|
|
precision := f.Precision()
|
|
|
|
if precision > Config.ModifyWindow {
|
|
|
|
Config.ModifyWindow = precision
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if Config.Verbose {
|
|
|
|
log.Printf("Modify window is %s\n", Config.ModifyWindow)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-12-31 17:31:19 +00:00
|
|
|
// Check the two files to see if the MD5sums are the same
|
|
|
|
//
|
|
|
|
// May return an error which will already have been logged
|
|
|
|
//
|
|
|
|
// If an error is returned it will return false
|
2013-06-28 07:57:32 +00:00
|
|
|
func CheckMd5sums(src, dst Object) (bool, error) {
|
2012-12-31 17:31:19 +00:00
|
|
|
srcMd5, err := src.Md5sum()
|
|
|
|
if err != nil {
|
2013-06-27 19:13:07 +00:00
|
|
|
Stats.Error()
|
2013-06-28 07:57:32 +00:00
|
|
|
Log(src, "Failed to calculate src md5: %s", err)
|
2012-12-31 17:31:19 +00:00
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
dstMd5, err := dst.Md5sum()
|
|
|
|
if err != nil {
|
2013-06-27 19:13:07 +00:00
|
|
|
Stats.Error()
|
2013-06-28 07:57:32 +00:00
|
|
|
Log(dst, "Failed to calculate dst md5: %s", err)
|
2012-12-31 17:31:19 +00:00
|
|
|
return false, err
|
|
|
|
}
|
2013-06-28 07:57:32 +00:00
|
|
|
// Debug("Src MD5 %s", srcMd5)
|
|
|
|
// Debug("Dst MD5 %s", obj.Hash)
|
2012-12-31 17:31:19 +00:00
|
|
|
return srcMd5 == dstMd5, nil
|
|
|
|
}
|
|
|
|
|
2012-12-26 12:23:58 +00:00
|
|
|
// Checks to see if the src and dst objects are equal by looking at
|
|
|
|
// size, mtime and MD5SUM
|
|
|
|
//
|
|
|
|
// If the src and dst size are different then it is considered to be
|
|
|
|
// not equal.
|
|
|
|
//
|
|
|
|
// If the size is the same and the mtime is the same then it is
|
|
|
|
// considered to be equal. This is the heuristic rsync uses when
|
|
|
|
// not using --checksum.
|
|
|
|
//
|
|
|
|
// If the size is the same and and mtime is different or unreadable
|
|
|
|
// and the MD5SUM is the same then the file is considered to be equal.
|
|
|
|
// In this case the mtime on the dst is updated.
|
|
|
|
//
|
|
|
|
// Otherwise the file is considered to be not equal including if there
|
|
|
|
// were errors reading info.
|
2013-06-28 07:57:32 +00:00
|
|
|
func Equal(src, dst Object) bool {
|
2012-12-26 12:23:58 +00:00
|
|
|
if src.Size() != dst.Size() {
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Sizes differ")
|
2012-12-26 12:23:58 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Size the same so check the mtime
|
2013-01-02 15:21:55 +00:00
|
|
|
srcModTime := src.ModTime()
|
|
|
|
dstModTime := dst.ModTime()
|
2013-01-18 23:21:02 +00:00
|
|
|
dt := dstModTime.Sub(srcModTime)
|
2013-06-27 19:13:07 +00:00
|
|
|
ModifyWindow := Config.ModifyWindow
|
|
|
|
if dt >= ModifyWindow || dt <= -ModifyWindow {
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
|
2012-12-26 12:23:58 +00:00
|
|
|
} else {
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
|
2013-01-02 15:21:55 +00:00
|
|
|
return true
|
2012-12-26 12:23:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// mtime is unreadable or different but size is the same so
|
|
|
|
// check the MD5SUM
|
2013-01-02 15:21:55 +00:00
|
|
|
same, _ := CheckMd5sums(src, dst)
|
2012-12-31 17:31:19 +00:00
|
|
|
if !same {
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Md5sums differ")
|
2012-12-26 12:23:58 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Size and MD5 the same but mtime different so update the
|
|
|
|
// mtime of the dst object here
|
|
|
|
dst.SetModTime(srcModTime)
|
|
|
|
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Size and MD5SUM of src and dst objects identical")
|
2012-12-26 12:23:58 +00:00
|
|
|
return true
|
|
|
|
}
|
2013-01-10 21:58:46 +00:00
|
|
|
|
|
|
|
// Copy src object to f
|
2013-06-28 07:57:32 +00:00
|
|
|
func Copy(f Fs, src Object) {
|
2013-01-10 21:58:46 +00:00
|
|
|
in0, err := src.Open()
|
|
|
|
if err != nil {
|
2013-06-27 19:13:07 +00:00
|
|
|
Stats.Error()
|
2013-06-28 07:57:32 +00:00
|
|
|
Log(src, "Failed to open: %s", err)
|
2013-01-10 21:58:46 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
in := NewAccount(in0) // account the transfer
|
|
|
|
|
|
|
|
dst, err := f.Put(in, src.Remote(), src.ModTime(), src.Size())
|
|
|
|
inErr := in.Close()
|
|
|
|
if err == nil {
|
|
|
|
err = inErr
|
|
|
|
}
|
|
|
|
if err != nil {
|
2013-06-27 19:13:07 +00:00
|
|
|
Stats.Error()
|
2013-06-28 07:57:32 +00:00
|
|
|
Log(src, "Failed to copy: %s", err)
|
2013-01-18 18:54:19 +00:00
|
|
|
if dst != nil {
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(dst, "Removing failed copy")
|
2013-01-18 18:54:19 +00:00
|
|
|
removeErr := dst.Remove()
|
|
|
|
if removeErr != nil {
|
2013-06-27 19:13:07 +00:00
|
|
|
Stats.Error()
|
2013-06-28 07:57:32 +00:00
|
|
|
Log(dst, "Failed to remove failed copy: %s", removeErr)
|
2013-01-18 18:54:19 +00:00
|
|
|
}
|
2013-01-10 21:58:46 +00:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2013-06-28 07:57:32 +00:00
|
|
|
Debug(src, "Copied")
|
2013-01-10 21:58:46 +00:00
|
|
|
}
|