diff --git a/drive/fs.go b/drive/fs.go index 629435fd1..6599efb99 100644 --- a/drive/fs.go +++ b/drive/fs.go @@ -289,7 +289,7 @@ func NewFsDrive(path string) (*FsDrive, error) { // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObject { +func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object { fs := &FsObjectDrive{ drive: f, remote: remote, @@ -299,7 +299,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje } else { err := fs.readMetaData() // reads info and meta, returning an error if err != nil { - // logged already fs.FsDebug("Failed to read info: %s", err) + // logged already fs.Debug("Failed to read info: %s", err) return nil } } @@ -309,7 +309,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsDrive) NewFsObject(remote string) fs.FsObject { +func (f *FsDrive) NewFsObject(remote string) fs.Object { return f.NewFsObjectWithInfo(remote, nil) } @@ -319,7 +319,7 @@ func (f *FsDrive) NewFsObject(remote string) fs.FsObject { // // This fetches the minimum amount of stuff but does more API calls // which makes it slow -func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsChan) error { +func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { var subError error // Make the API request _, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool { @@ -357,7 +357,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsCh // // This is fast in terms of number of API calls, but slow in terms of // fetching more data than it needs -func (f *FsDrive) listDirFull(dirId string, path string, out fs.FsObjectsChan) error { +func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) error { // Orphans waiting for their parent orphans := make(map[string][]*drive.File) @@ -547,8 +547,8 @@ func (f *FsDrive) findRoot(create bool) error { } // Walk the path returning a channel of FsObjects -func (f *FsDrive) List() fs.FsObjectsChan { - out := make(fs.FsObjectsChan, fs.Config.Checkers) +func (f *FsDrive) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) go func() { defer close(out) err := f.findRoot(false) @@ -571,8 +571,8 @@ func (f *FsDrive) List() fs.FsObjectsChan { } // Walk the path returning a channel of FsObjects -func (f *FsDrive) ListDir() fs.FsDirChan { - out := make(fs.FsDirChan, fs.Config.Checkers) +func (f *FsDrive) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) go func() { defer close(out) err := f.findRoot(false) @@ -581,7 +581,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan { log.Printf("Couldn't find root: %s", err) } else { _, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool { - dir := &fs.FsDir{ + dir := &fs.Dir{ Name: item.Title, Bytes: -1, Count: -1, @@ -604,7 +604,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan { // Copy the reader in to the new object which is returned // // The new object may have been created -func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { +func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectDrive{drive: f, remote: remote} @@ -744,7 +744,7 @@ func (o *FsObjectDrive) readMetaData() (err error) { directory, leaf := splitPath(o.remote) directoryId, err := o.drive.findDir(directory, false) if err != nil { - fs.FsDebug(o, "Couldn't find directory: %s", err) + fs.Debug(o, "Couldn't find directory: %s", err) return fmt.Errorf("Couldn't find directory: %s", err) } @@ -759,7 +759,7 @@ func (o *FsObjectDrive) readMetaData() (err error) { return err } if !found { - fs.FsDebug(o, "Couldn't find object") + fs.Debug(o, "Couldn't find object") return fmt.Errorf("Couldn't find object") } return nil @@ -773,12 +773,12 @@ func (o *FsObjectDrive) readMetaData() (err error) { func (o *FsObjectDrive) ModTime() time.Time { err := o.readMetaData() if err != nil { - fs.FsLog(o, "Failed to read metadata: %s", err) + fs.Log(o, "Failed to read metadata: %s", err) return time.Now() } modTime, err := time.Parse(time.RFC3339, o.modifiedDate) if err != nil { - fs.FsLog(o, "Failed to read mtime from object: %s", err) + fs.Log(o, "Failed to read mtime from object: %s", err) return time.Now() } return modTime @@ -789,7 +789,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to read metadata: %s", err) + fs.Log(o, "Failed to read metadata: %s", err) return } // New metadata @@ -800,7 +800,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) { _, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do() if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to update remote mtime: %s", err) + fs.Log(o, "Failed to update remote mtime: %s", err) } } @@ -832,4 +832,4 @@ func (o *FsObjectDrive) Remove() error { // Check the interfaces are satisfied var _ fs.Fs = &FsDrive{} var _ fs.Purger = &FsDrive{} -var _ fs.FsObject = &FsObjectDrive{} +var _ fs.Object = &FsObjectDrive{} diff --git a/fs/accounting.go b/fs/accounting.go index 9af5a4caa..a5c455f97 100644 --- a/fs/accounting.go +++ b/fs/accounting.go @@ -121,32 +121,32 @@ func (s *StatsInfo) Error() { } // Checking adds a check into the stats -func (s *StatsInfo) Checking(fs FsObject) { +func (s *StatsInfo) Checking(o Object) { s.lock.Lock() defer s.lock.Unlock() - s.checking[fs.Remote()] = true + s.checking[o.Remote()] = true } // DoneChecking removes a check from the stats -func (s *StatsInfo) DoneChecking(fs FsObject) { +func (s *StatsInfo) DoneChecking(o Object) { s.lock.Lock() defer s.lock.Unlock() - delete(s.checking, fs.Remote()) + delete(s.checking, o.Remote()) s.checks += 1 } // Transferring adds a transfer into the stats -func (s *StatsInfo) Transferring(fs FsObject) { +func (s *StatsInfo) Transferring(o Object) { s.lock.Lock() defer s.lock.Unlock() - s.transferring[fs.Remote()] = true + s.transferring[o.Remote()] = true } // DoneTransferring removes a transfer from the stats -func (s *StatsInfo) DoneTransferring(fs FsObject) { +func (s *StatsInfo) DoneTransferring(o Object) { s.lock.Lock() defer s.lock.Unlock() - delete(s.transferring, fs.Remote()) + delete(s.transferring, o.Remote()) s.transfers += 1 } diff --git a/fs/fs.go b/fs/fs.go index 2b574cbec..3446b77a1 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -13,24 +13,24 @@ import ( // Globals var ( // Global config - Config = &FsConfig{} + Config = &ConfigInfo{} // Filesystem registry - fsRegistry []fsRegistryItem + fsRegistry []registryItem ) // Filesystem config options -type FsConfig struct { - Verbose bool - Quiet bool +type ConfigInfo struct { + Verbose bool + Quiet bool ModifyWindow time.Duration - Checkers int - Transfers int + Checkers int + Transfers int } // FIXME need local to go last // Filesystem registry item -type fsRegistryItem struct { +type registryItem struct { match *regexp.Regexp // if this matches then can call newFs newFs func(string) (Fs, error) // create a new file system } @@ -41,7 +41,7 @@ type fsRegistryItem struct { // // Fs modules should use this in an init() function func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) { - fsRegistry = append(fsRegistry, fsRegistryItem{match: match, newFs: newFs}) + fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs}) } // A Filesystem, describes the local filesystem and the remote object store @@ -50,20 +50,20 @@ type Fs interface { String() string // List the Fs into a channel - List() FsObjectsChan + List() ObjectsChan // List the Fs directories/buckets/containers into a channel - ListDir() FsDirChan + ListDir() DirChan - // Find the FsObject at remote. Returns nil if can't be found - NewFsObject(remote string) FsObject + // Find the Object at remote. Returns nil if can't be found + NewFsObject(remote string) Object // Put in to the remote path with the modTime given of the given size // // May create the object even if it returns an error - if so // will return the object and the error, otherwise will return // nil and the error - Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) + Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) // Make the directory (container, bucket) Mkdir() error @@ -79,7 +79,7 @@ type Fs interface { // A filesystem like object which can either be a remote object or a // local file/directory -type FsObject interface { +type Object interface { // Remote returns the remote path Remote() string @@ -114,22 +114,22 @@ type Purger interface { Purge() error } -// A channel of FsObjects -type FsObjectsChan chan FsObject +// A channel of Objects +type ObjectsChan chan Object -// A slice of FsObjects -type FsObjects []FsObject +// A slice of Objects +type Objects []Object // A structure of directory/container/bucket lists -type FsDir struct { +type Dir struct { Name string // name of the directory When time.Time // modification or creation time - IsZero for unknown Bytes int64 // size of directory and contents -1 for unknown Count int64 // number of objects -1 for unknown } -// A channel of FsDir objects -type FsDirChan chan *FsDir +// A channel of Dir objects +type DirChan chan *Dir // NewFs makes a new Fs object from the path // @@ -143,16 +143,16 @@ func NewFs(path string) (Fs, error) { panic("Not found") // FIXME } -// Write debuging output for this FsObject -func FsDebug(fs FsObject, text string, args ...interface{}) { +// Write debuging output for this Object +func Debug(fs Object, text string, args ...interface{}) { if Config.Verbose { out := fmt.Sprintf(text, args...) log.Printf("%s: %s", fs.Remote(), out) } } -// Write log output for this FsObject -func FsLog(fs FsObject, text string, args ...interface{}) { +// Write log output for this Object +func Log(fs Object, text string, args ...interface{}) { if !Config.Quiet { out := fmt.Sprintf(text, args...) log.Printf("%s: %s", fs.Remote(), out) @@ -173,21 +173,21 @@ func checkClose(c io.Closer, err *error) { // May return an error which will already have been logged // // If an error is returned it will return false -func CheckMd5sums(src, dst FsObject) (bool, error) { +func CheckMd5sums(src, dst Object) (bool, error) { srcMd5, err := src.Md5sum() if err != nil { Stats.Error() - FsLog(src, "Failed to calculate src md5: %s", err) + Log(src, "Failed to calculate src md5: %s", err) return false, err } dstMd5, err := dst.Md5sum() if err != nil { Stats.Error() - FsLog(dst, "Failed to calculate dst md5: %s", err) + Log(dst, "Failed to calculate dst md5: %s", err) return false, err } - // FsDebug("Src MD5 %s", srcMd5) - // FsDebug("Dst MD5 %s", obj.Hash) + // Debug("Src MD5 %s", srcMd5) + // Debug("Dst MD5 %s", obj.Hash) return srcMd5 == dstMd5, nil } @@ -207,9 +207,9 @@ func CheckMd5sums(src, dst FsObject) (bool, error) { // // Otherwise the file is considered to be not equal including if there // were errors reading info. -func Equal(src, dst FsObject) bool { +func Equal(src, dst Object) bool { if src.Size() != dst.Size() { - FsDebug(src, "Sizes differ") + Debug(src, "Sizes differ") return false } @@ -219,9 +219,9 @@ func Equal(src, dst FsObject) bool { dt := dstModTime.Sub(srcModTime) ModifyWindow := Config.ModifyWindow if dt >= ModifyWindow || dt <= -ModifyWindow { - FsDebug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime) + Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime) } else { - FsDebug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow) + Debug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow) return true } @@ -229,7 +229,7 @@ func Equal(src, dst FsObject) bool { // check the MD5SUM same, _ := CheckMd5sums(src, dst) if !same { - FsDebug(src, "Md5sums differ") + Debug(src, "Md5sums differ") return false } @@ -237,16 +237,16 @@ func Equal(src, dst FsObject) bool { // mtime of the dst object here dst.SetModTime(srcModTime) - FsDebug(src, "Size and MD5SUM of src and dst objects identical") + Debug(src, "Size and MD5SUM of src and dst objects identical") return true } // Copy src object to f -func Copy(f Fs, src FsObject) { +func Copy(f Fs, src Object) { in0, err := src.Open() if err != nil { Stats.Error() - FsLog(src, "Failed to open: %s", err) + Log(src, "Failed to open: %s", err) return } in := NewAccount(in0) // account the transfer @@ -258,16 +258,16 @@ func Copy(f Fs, src FsObject) { } if err != nil { Stats.Error() - FsLog(src, "Failed to copy: %s", err) + Log(src, "Failed to copy: %s", err) if dst != nil { - FsDebug(dst, "Removing failed copy") + Debug(dst, "Removing failed copy") removeErr := dst.Remove() if removeErr != nil { Stats.Error() - FsLog(dst, "Failed to remove failed copy: %s", removeErr) + Log(dst, "Failed to remove failed copy: %s", removeErr) } } return } - FsDebug(src, "Copied") + Debug(src, "Copied") } diff --git a/local/fs.go b/local/fs.go index 5f8afdc11..e7bbed7f8 100644 --- a/local/fs.go +++ b/local/fs.go @@ -55,7 +55,7 @@ func (f *FsLocal) String() string { // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObject { +func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object { path := filepath.Join(f.root, remote) o := &FsObjectLocal{remote: remote, path: path} if info != nil { @@ -63,7 +63,7 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje } else { err := o.lstat() if err != nil { - fs.FsDebug(o, "Failed to stat %s: %s", path, err) + fs.Debug(o, "Failed to stat %s: %s", path, err) return nil } } @@ -73,15 +73,15 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsLocal) NewFsObject(remote string) fs.FsObject { +func (f *FsLocal) NewFsObject(remote string) fs.Object { return f.NewFsObjectWithInfo(remote, nil) } // List the path returning a channel of FsObjects // // Ignores everything which isn't Storable, eg links etc -func (f *FsLocal) List() fs.FsObjectsChan { - out := make(fs.FsObjectsChan, fs.Config.Checkers) +func (f *FsLocal) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) go func() { err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -116,8 +116,8 @@ func (f *FsLocal) List() fs.FsObjectsChan { } // Walk the path returning a channel of FsObjects -func (f *FsLocal) ListDir() fs.FsDirChan { - out := make(fs.FsDirChan, fs.Config.Checkers) +func (f *FsLocal) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) go func() { defer close(out) items, err := ioutil.ReadDir(f.root) @@ -127,7 +127,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan { } else { for _, item := range items { if item.IsDir() { - dir := &fs.FsDir{ + dir := &fs.Dir{ Name: item.Name(), When: item.ModTime(), Bytes: 0, @@ -159,7 +159,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan { } // Puts the FsObject to the local filesystem -func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { +func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { dstPath := filepath.Join(f.root, remote) // Temporary FsObject under construction fs := &FsObjectLocal{remote: remote, path: dstPath} @@ -270,7 +270,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) { in, err := os.Open(o.path) if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to open: %s", err) + fs.Log(o, "Failed to open: %s", err) return "", err } defer in.Close() // FIXME ignoring error @@ -278,7 +278,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) { _, err = io.Copy(hash, in) if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to read: %s", err) + fs.Log(o, "Failed to read: %s", err) return "", err } return fmt.Sprintf("%x", hash.Sum(nil)), nil @@ -298,7 +298,7 @@ func (o *FsObjectLocal) ModTime() time.Time { func (o *FsObjectLocal) SetModTime(modTime time.Time) { err := os.Chtimes(o.path, modTime, modTime) if err != nil { - fs.FsDebug(o, "Failed to set mtime on file: %s", err) + fs.Debug(o, "Failed to set mtime on file: %s", err) } } @@ -306,10 +306,10 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) { func (o *FsObjectLocal) Storable() bool { mode := o.info.Mode() if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { - fs.FsDebug(o, "Can't transfer non file/directory") + fs.Debug(o, "Can't transfer non file/directory") return false } else if mode&os.ModeDir != 0 { - fs.FsDebug(o, "FIXME Skipping directory") + fs.Debug(o, "FIXME Skipping directory") return false } return true @@ -335,4 +335,4 @@ func (o *FsObjectLocal) Remove() error { // Check the interfaces are satisfied var _ fs.Fs = &FsLocal{} -var _ fs.FsObject = &FsObjectLocal{} +var _ fs.Object = &FsObjectLocal{} diff --git a/rclone.go b/rclone.go index 2a94340c6..5124ad755 100644 --- a/rclone.go +++ b/rclone.go @@ -34,17 +34,17 @@ var ( modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same") ) -// A pair of fs.FsObjects +// A pair of fs.Objects type PairFsObjects struct { - src, dst fs.FsObject + src, dst fs.Object } type PairFsObjectsChan chan PairFsObjects // Check to see if src needs to be copied to dst and if so puts it in out -func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) { +func checkOne(src, dst fs.Object, out fs.ObjectsChan) { if dst == nil { - fs.FsDebug(src, "Couldn't find local file - download") + fs.Debug(src, "Couldn't find local file - download") out <- src return } @@ -54,7 +54,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) { } // Check to see if changed or not if fs.Equal(src, dst) { - fs.FsDebug(src, "Unchanged skipping") + fs.Debug(src, "Unchanged skipping") return } out <- src @@ -63,7 +63,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) { // Read FsObjects~s on in send to out if they need uploading // // FIXME potentially doing lots of MD5SUMS at once -func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup) { +func PairChecker(in PairFsObjectsChan, out fs.ObjectsChan, wg *sync.WaitGroup) { defer wg.Done() for pair := range in { src := pair.src @@ -76,7 +76,7 @@ func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup) // Read FsObjects~s on in send to out if they need uploading // // FIXME potentially doing lots of MD5SUMS at once -func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { +func Checker(in, out fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { defer wg.Done() for src := range in { fs.Stats.Checking(src) @@ -87,7 +87,7 @@ func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { } // Read FsObjects on in and copy them -func Copier(in fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { +func Copier(in fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { defer wg.Done() for src := range in { fs.Stats.Transferring(src) @@ -105,7 +105,7 @@ func CopyFs(fdst, fsrc fs.Fs) { } to_be_checked := fsrc.List() - to_be_uploaded := make(fs.FsObjectsChan, *transfers) + to_be_uploaded := make(fs.ObjectsChan, *transfers) var checkerWg sync.WaitGroup checkerWg.Add(*checkers) @@ -127,7 +127,7 @@ func CopyFs(fdst, fsrc fs.Fs) { } // Delete all the files passed in the channel -func DeleteFiles(to_be_deleted fs.FsObjectsChan) { +func DeleteFiles(to_be_deleted fs.ObjectsChan) { var wg sync.WaitGroup wg.Add(*transfers) for i := 0; i < *transfers; i++ { @@ -135,16 +135,16 @@ func DeleteFiles(to_be_deleted fs.FsObjectsChan) { defer wg.Done() for dst := range to_be_deleted { if *dry_run { - fs.FsDebug(dst, "Not deleting as -dry-run") + fs.Debug(dst, "Not deleting as -dry-run") } else { fs.Stats.Checking(dst) err := dst.Remove() fs.Stats.DoneChecking(dst) if err != nil { fs.Stats.Error() - fs.FsLog(dst, "Couldn't delete: %s", err) + fs.Log(dst, "Couldn't delete: %s", err) } else { - fs.FsDebug(dst, "Deleted") + fs.Debug(dst, "Deleted") } } } @@ -167,14 +167,14 @@ func Sync(fdst, fsrc fs.Fs) { // Read the destination files first // FIXME could do this in parallel and make it use less memory - delFiles := make(map[string]fs.FsObject) + delFiles := make(map[string]fs.Object) for dst := range fdst.List() { delFiles[dst.Remote()] = dst } // Read source files checking them off against dest files to_be_checked := make(PairFsObjectsChan, *transfers) - to_be_uploaded := make(fs.FsObjectsChan, *transfers) + to_be_uploaded := make(fs.ObjectsChan, *transfers) var checkerWg sync.WaitGroup checkerWg.Add(*checkers) @@ -215,7 +215,7 @@ func Sync(fdst, fsrc fs.Fs) { } // Delete the spare files - toDelete := make(fs.FsObjectsChan, *transfers) + toDelete := make(fs.ObjectsChan, *transfers) go func() { for _, fs := range delFiles { toDelete <- fs @@ -231,19 +231,19 @@ func Check(fdst, fsrc fs.Fs) { // Read the destination files first // FIXME could do this in parallel and make it use less memory - dstFiles := make(map[string]fs.FsObject) + dstFiles := make(map[string]fs.Object) for dst := range fdst.List() { dstFiles[dst.Remote()] = dst } // Read the source files checking them against dstFiles // FIXME could do this in parallel and make it use less memory - srcFiles := make(map[string]fs.FsObject) - commonFiles := make(map[string][]fs.FsObject) + srcFiles := make(map[string]fs.Object) + commonFiles := make(map[string][]fs.Object) for src := range fsrc.List() { remote := src.Remote() if dst, ok := dstFiles[remote]; ok { - commonFiles[remote] = []fs.FsObject{dst, src} + commonFiles[remote] = []fs.Object{dst, src} delete(dstFiles, remote) } else { srcFiles[remote] = src @@ -262,7 +262,7 @@ func Check(fdst, fsrc fs.Fs) { log.Printf(remote) } - checks := make(chan []fs.FsObject, *transfers) + checks := make(chan []fs.Object, *transfers) go func() { for _, check := range commonFiles { checks <- check @@ -281,7 +281,7 @@ func Check(fdst, fsrc fs.Fs) { if src.Size() != dst.Size() { fs.Stats.DoneChecking(src) fs.Stats.Error() - fs.FsLog(src, "Sizes differ") + fs.Log(src, "Sizes differ") continue } same, err := fs.CheckMd5sums(src, dst) @@ -291,9 +291,9 @@ func Check(fdst, fsrc fs.Fs) { } if !same { fs.Stats.Error() - fs.FsLog(src, "Md5sums differ") + fs.Log(src, "Md5sums differ") } - fs.FsDebug(src, "OK") + fs.Debug(src, "OK") } }() } diff --git a/s3/fs.go b/s3/fs.go index 93a1764ac..4d732350c 100644 --- a/s3/fs.go +++ b/s3/fs.go @@ -143,7 +143,7 @@ func NewFs(path string) (fs.Fs, error) { // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject { +func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object { o := &FsObjectS3{ s3: f, remote: remote, @@ -153,7 +153,7 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject { var err error o.lastModified, err = time.Parse(time.RFC3339, info.LastModified) if err != nil { - fs.FsLog(o, "Failed to read last modified: %s", err) + fs.Log(o, "Failed to read last modified: %s", err) o.lastModified = time.Now() } o.etag = info.ETag @@ -171,13 +171,13 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject { // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsS3) NewFsObject(remote string) fs.FsObject { +func (f *FsS3) NewFsObject(remote string) fs.Object { return f.NewFsObjectWithInfo(remote, nil) } // Walk the path returning a channel of FsObjects -func (f *FsS3) List() fs.FsObjectsChan { - out := make(fs.FsObjectsChan, fs.Config.Checkers) +func (f *FsS3) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) go func() { // FIXME need to implement ALL loop objects, err := f.b.List("", "", "", 10000) @@ -198,8 +198,8 @@ func (f *FsS3) List() fs.FsObjectsChan { } // Lists the buckets -func (f *FsS3) ListDir() fs.FsDirChan { - out := make(fs.FsDirChan, fs.Config.Checkers) +func (f *FsS3) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) go func() { defer close(out) buckets, err := f.c.ListBuckets() @@ -208,7 +208,7 @@ func (f *FsS3) ListDir() fs.FsDirChan { log.Printf("Couldn't list buckets: %s", err) } else { for _, bucket := range buckets { - out <- &fs.FsDir{ + out <- &fs.Dir{ Name: bucket.Name, When: bucket.CreationDate, Bytes: -1, @@ -221,7 +221,7 @@ func (f *FsS3) ListDir() fs.FsDirChan { } // Put the FsObject into the bucket -func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { +func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectS3{s3: f, remote: remote} @@ -290,19 +290,19 @@ func (o *FsObjectS3) readMetaData() (err error) { headers, err := o.s3.b.Head(o.remote, nil) if err != nil { - fs.FsDebug(o, "Failed to read info: %s", err) + fs.Debug(o, "Failed to read info: %s", err) return err } size, err := strconv.ParseInt(headers["Content-Length"], 10, 64) if err != nil { - fs.FsDebug(o, "Failed to read size from: %q", headers) + fs.Debug(o, "Failed to read size from: %q", headers) return err } o.etag = headers["Etag"] o.bytes = size o.meta = headers if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil { - fs.FsLog(o, "Failed to read last modified from HEAD: %s", err) + fs.Log(o, "Failed to read last modified from HEAD: %s", err) o.lastModified = time.Now() } return nil @@ -315,18 +315,18 @@ func (o *FsObjectS3) readMetaData() (err error) { func (o *FsObjectS3) ModTime() time.Time { err := o.readMetaData() if err != nil { - fs.FsLog(o, "Failed to read metadata: %s", err) + fs.Log(o, "Failed to read metadata: %s", err) return time.Now() } // read mtime out of metadata if available d, ok := o.meta[metaMtime] if !ok { - // fs.FsDebug(o, "No metadata") + // fs.Debug(o, "No metadata") return o.lastModified } modTime, err := swift.FloatStringToTime(d) if err != nil { - fs.FsLog(o, "Failed to read mtime from object: %s", err) + fs.Log(o, "Failed to read mtime from object: %s", err) return o.lastModified } return modTime @@ -337,14 +337,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to read metadata: %s", err) + fs.Log(o, "Failed to read metadata: %s", err) return } o.meta[metaMtime] = swift.TimeToFloatString(modTime) _, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta) if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to update remote mtime: %s", err) + fs.Log(o, "Failed to update remote mtime: %s", err) } } @@ -366,4 +366,4 @@ func (o *FsObjectS3) Remove() error { // Check the interfaces are satisfied var _ fs.Fs = &FsS3{} -var _ fs.FsObject = &FsObjectS3{} +var _ fs.Object = &FsObjectS3{} diff --git a/swift/fs.go b/swift/fs.go index 6c12cc22a..824d9520d 100644 --- a/swift/fs.go +++ b/swift/fs.go @@ -114,7 +114,7 @@ func NewFs(path string) (fs.Fs, error) { // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsObject { +func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object { fs := &FsObjectSwift{ swift: f, remote: remote, @@ -135,13 +135,13 @@ func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsOb // Return an FsObject from a path // // May return nil if an error occurred -func (f *FsSwift) NewFsObject(remote string) fs.FsObject { +func (f *FsSwift) NewFsObject(remote string) fs.Object { return f.NewFsObjectWithInfo(remote, nil) } // Walk the path returning a channel of FsObjects -func (f *FsSwift) List() fs.FsObjectsChan { - out := make(fs.FsObjectsChan, fs.Config.Checkers) +func (f *FsSwift) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) go func() { // FIXME use a smaller limit? err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) { @@ -166,8 +166,8 @@ func (f *FsSwift) List() fs.FsObjectsChan { } // Lists the containers -func (f *FsSwift) ListDir() fs.FsDirChan { - out := make(fs.FsDirChan, fs.Config.Checkers) +func (f *FsSwift) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) go func() { defer close(out) containers, err := f.c.ContainersAll(nil) @@ -176,7 +176,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan { log.Printf("Couldn't list containers: %s", err) } else { for _, container := range containers { - out <- &fs.FsDir{ + out <- &fs.Dir{ Name: container.Name, Bytes: container.Bytes, Count: container.Count, @@ -192,7 +192,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan { // Copy the reader in to the new object which is returned // // The new object may have been created -func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { +func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { // Temporary FsObject under construction fs := &FsObjectSwift{swift: f, remote: remote} @@ -246,7 +246,7 @@ func (o *FsObjectSwift) readMetaData() (err error) { } info, h, err := o.swift.c.Object(o.swift.container, o.remote) if err != nil { - fs.FsDebug(o, "Failed to read info: %s", err) + fs.Debug(o, "Failed to read info: %s", err) return err } meta := h.ObjectMetadata() @@ -263,12 +263,12 @@ func (o *FsObjectSwift) readMetaData() (err error) { func (o *FsObjectSwift) ModTime() time.Time { err := o.readMetaData() if err != nil { - // fs.FsLog(o, "Failed to read metadata: %s", err) + // fs.Log(o, "Failed to read metadata: %s", err) return o.info.LastModified } modTime, err := o.meta.GetModTime() if err != nil { - // fs.FsLog(o, "Failed to read mtime from object: %s", err) + // fs.Log(o, "Failed to read mtime from object: %s", err) return o.info.LastModified } return modTime @@ -279,14 +279,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) { err := o.readMetaData() if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to read metadata: %s", err) + fs.Log(o, "Failed to read metadata: %s", err) return } o.meta.SetModTime(modTime) err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders()) if err != nil { fs.Stats.Error() - fs.FsLog(o, "Failed to update remote mtime: %s", err) + fs.Log(o, "Failed to update remote mtime: %s", err) } } @@ -308,4 +308,4 @@ func (o *FsObjectSwift) Remove() error { // Check the interfaces are satisfied var _ fs.Fs = &FsSwift{} -var _ fs.FsObject = &FsObjectSwift{} +var _ fs.Object = &FsObjectSwift{}