rclone/fs/fs.go

274 lines
6.7 KiB
Go
Raw Normal View History

// File system interface
2013-06-27 19:13:07 +00:00
package fs
import (
"fmt"
"io"
"log"
2013-06-27 19:13:07 +00:00
"regexp"
"time"
)
2013-06-27 19:13:07 +00:00
// Globals
var (
// Global config
Config = &FsConfig{}
// Filesystem registry
fsRegistry []fsRegistryItem
)
// Filesystem config options
type FsConfig struct {
Verbose bool
Quiet bool
ModifyWindow time.Duration
Checkers int
Transfers int
}
// FIXME need local to go last
// Filesystem registry item
type fsRegistryItem struct {
match *regexp.Regexp // if this matches then can call newFs
newFs func(string) (Fs, error) // create a new file system
}
// Register a filesystem
//
// If a path matches with match then can call newFs on it
//
// Fs modules should use this in an init() function
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
fsRegistry = append(fsRegistry, fsRegistryItem{match: match, newFs: newFs})
}
// A Filesystem, describes the local filesystem and the remote object store
type Fs interface {
2013-01-18 18:54:19 +00:00
// String returns a description of the FS
String() string
2013-01-18 18:54:19 +00:00
// List the Fs into a channel
List() FsObjectsChan
2013-01-18 18:54:19 +00:00
2013-01-23 22:43:20 +00:00
// List the Fs directories/buckets/containers into a channel
ListDir() FsDirChan
2013-01-18 18:54:19 +00:00
// Find the FsObject at remote. Returns nil if can't be found
NewFsObject(remote string) FsObject
2013-01-18 18:54:19 +00:00
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error)
2013-01-18 18:54:19 +00:00
// Make the directory (container, bucket)
Mkdir() error
2013-01-18 18:54:19 +00:00
// Remove the directory (container, bucket) if empty
Rmdir() error
// Precision of the ModTimes in this Fs
Precision() time.Duration
}
// FIXME make f.Debugf...
// A filesystem like object which can either be a remote object or a
// local file/directory
type FsObject interface {
2013-01-18 18:54:19 +00:00
// Remote returns the remote path
Remote() string
2013-01-18 18:54:19 +00:00
// Md5sum returns the md5 checksum of the file
Md5sum() (string, error)
2013-01-18 18:54:19 +00:00
// ModTime returns the modification date of the file
ModTime() time.Time
2013-01-18 18:54:19 +00:00
// SetModTime sets the metadata on the object to set the modification date
SetModTime(time.Time)
2013-01-18 18:54:19 +00:00
// Size returns the size of the file
Size() int64
2013-01-18 18:54:19 +00:00
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error)
2013-01-18 18:54:19 +00:00
// Storable says whether this object can be stored
Storable() bool
2013-01-18 18:54:19 +00:00
// Removes this object
Remove() error
}
// Optional interfaces
type Purger interface {
// Purge all files in the root and the root directory
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
Purge() error
}
2013-01-18 18:54:19 +00:00
// A channel of FsObjects
type FsObjectsChan chan FsObject
2013-01-18 18:54:19 +00:00
// A slice of FsObjects
type FsObjects []FsObject
2013-01-23 22:43:20 +00:00
// A structure of directory/container/bucket lists
type FsDir struct {
Name string // name of the directory
When time.Time // modification or creation time - IsZero for unknown
Bytes int64 // size of directory and contents -1 for unknown
Count int64 // number of objects -1 for unknown
}
// A channel of FsDir objects
type FsDirChan chan *FsDir
// NewFs makes a new Fs object from the path
//
2013-01-18 18:54:19 +00:00
// FIXME make more generic
func NewFs(path string) (Fs, error) {
2013-06-27 19:13:07 +00:00
for _, item := range fsRegistry {
if item.match.MatchString(path) {
return item.newFs(path)
}
}
2013-06-27 19:13:07 +00:00
panic("Not found") // FIXME
}
// Write debuging output for this FsObject
func FsDebug(fs FsObject, text string, args ...interface{}) {
2013-06-27 19:13:07 +00:00
if Config.Verbose {
out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out)
}
}
// Write log output for this FsObject
func FsLog(fs FsObject, text string, args ...interface{}) {
2013-06-27 19:13:07 +00:00
if !Config.Quiet {
out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out)
}
}
// checkClose is a utility function used to check the return from
// Close in a defer statement.
func checkClose(c io.Closer, err *error) {
cerr := c.Close()
if *err == nil {
*err = cerr
}
}
2012-12-31 17:31:19 +00:00
// Check the two files to see if the MD5sums are the same
//
// May return an error which will already have been logged
//
// If an error is returned it will return false
func CheckMd5sums(src, dst FsObject) (bool, error) {
srcMd5, err := src.Md5sum()
if err != nil {
2013-06-27 19:13:07 +00:00
Stats.Error()
2012-12-31 17:31:19 +00:00
FsLog(src, "Failed to calculate src md5: %s", err)
return false, err
}
dstMd5, err := dst.Md5sum()
if err != nil {
2013-06-27 19:13:07 +00:00
Stats.Error()
2012-12-31 17:31:19 +00:00
FsLog(dst, "Failed to calculate dst md5: %s", err)
return false, err
}
// FsDebug("Src MD5 %s", srcMd5)
// FsDebug("Dst MD5 %s", obj.Hash)
return srcMd5 == dstMd5, nil
}
// Checks to see if the src and dst objects are equal by looking at
// size, mtime and MD5SUM
//
// If the src and dst size are different then it is considered to be
// not equal.
//
// If the size is the same and the mtime is the same then it is
// considered to be equal. This is the heuristic rsync uses when
// not using --checksum.
//
// If the size is the same and and mtime is different or unreadable
// and the MD5SUM is the same then the file is considered to be equal.
// In this case the mtime on the dst is updated.
//
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(src, dst FsObject) bool {
if src.Size() != dst.Size() {
FsDebug(src, "Sizes differ")
return false
}
// Size the same so check the mtime
srcModTime := src.ModTime()
dstModTime := dst.ModTime()
dt := dstModTime.Sub(srcModTime)
2013-06-27 19:13:07 +00:00
ModifyWindow := Config.ModifyWindow
if dt >= ModifyWindow || dt <= -ModifyWindow {
FsDebug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
} else {
2013-06-27 19:13:07 +00:00
FsDebug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
return true
}
// mtime is unreadable or different but size is the same so
// check the MD5SUM
same, _ := CheckMd5sums(src, dst)
2012-12-31 17:31:19 +00:00
if !same {
FsDebug(src, "Md5sums differ")
return false
}
// Size and MD5 the same but mtime different so update the
// mtime of the dst object here
dst.SetModTime(srcModTime)
FsDebug(src, "Size and MD5SUM of src and dst objects identical")
return true
}
// Copy src object to f
func Copy(f Fs, src FsObject) {
in0, err := src.Open()
if err != nil {
2013-06-27 19:13:07 +00:00
Stats.Error()
FsLog(src, "Failed to open: %s", err)
return
}
in := NewAccount(in0) // account the transfer
dst, err := f.Put(in, src.Remote(), src.ModTime(), src.Size())
inErr := in.Close()
if err == nil {
err = inErr
}
if err != nil {
2013-06-27 19:13:07 +00:00
Stats.Error()
2013-01-18 18:54:19 +00:00
FsLog(src, "Failed to copy: %s", err)
if dst != nil {
FsDebug(dst, "Removing failed copy")
removeErr := dst.Remove()
if removeErr != nil {
2013-06-27 19:13:07 +00:00
Stats.Error()
2013-01-18 18:54:19 +00:00
FsLog(dst, "Failed to remove failed copy: %s", removeErr)
}
}
return
}
FsDebug(src, "Copied")
}