2014-03-28 17:56:04 +00:00
// Generic operations on filesystems and objects
package fs
import (
"fmt"
2014-08-01 16:58:39 +00:00
"io"
2016-03-05 16:10:51 +00:00
"log"
2015-03-01 12:38:31 +00:00
"mime"
"path"
2016-03-05 16:10:51 +00:00
"sort"
2016-01-23 20:16:47 +00:00
"strings"
2014-03-28 17:56:04 +00:00
"sync"
2015-10-02 18:48:48 +00:00
"sync/atomic"
2016-01-23 20:16:47 +00:00
2016-06-12 14:06:02 +00:00
"github.com/pkg/errors"
2016-10-31 14:45:52 +00:00
"github.com/spf13/pflag"
2016-06-12 14:06:02 +00:00
2016-01-23 20:16:47 +00:00
"golang.org/x/text/unicode/norm"
2014-03-28 17:56:04 +00:00
)
2015-09-22 17:47:16 +00:00
// CalculateModifyWindow works out modify window for Fses passed in -
// sets Config.ModifyWindow
2014-03-28 17:56:04 +00:00
//
// This is the largest modify window of all the fses in use, and the
// user configured value
func CalculateModifyWindow ( fs ... Fs ) {
for _ , f := range fs {
if f != nil {
precision := f . Precision ( )
if precision > Config . ModifyWindow {
Config . ModifyWindow = precision
}
2015-08-20 19:48:58 +00:00
if precision == ModTimeNotSupported {
Debug ( f , "Modify window not supported" )
return
}
2014-03-28 17:56:04 +00:00
}
}
2015-08-20 19:48:58 +00:00
Debug ( fs [ 0 ] , "Modify window is %s" , Config . ModifyWindow )
2014-03-28 17:56:04 +00:00
}
2016-01-11 12:39:33 +00:00
// HashEquals checks to see if src == dst, but ignores empty strings
// and returns true if either is empty.
func HashEquals ( src , dst string ) bool {
2015-08-16 22:24:34 +00:00
if src == "" || dst == "" {
return true
}
return src == dst
}
2016-01-11 12:39:33 +00:00
// CheckHashes checks the two files to see if they have common
// known hash types and compares them
2014-03-28 17:56:04 +00:00
//
2016-01-24 18:06:57 +00:00
// Returns
2015-08-20 19:48:58 +00:00
//
2016-01-24 18:06:57 +00:00
// equal - which is equality of the hashes
//
// hash - the HashType. This is HashNone if either of the hashes were
// unset or a compatible hash couldn't be found.
//
// err - may return an error which will already have been logged
2014-03-28 17:56:04 +00:00
//
2015-08-20 19:48:58 +00:00
// If an error is returned it will return equal as false
2016-01-24 18:06:57 +00:00
func CheckHashes ( src , dst Object ) ( equal bool , hash HashType , err error ) {
2016-01-11 12:39:33 +00:00
common := src . Fs ( ) . Hashes ( ) . Overlap ( dst . Fs ( ) . Hashes ( ) )
2016-01-24 18:06:57 +00:00
// Debug(nil, "Shared hashes: %v", common)
2016-01-11 12:39:33 +00:00
if common . Count ( ) == 0 {
2016-01-24 18:06:57 +00:00
return true , HashNone , nil
2016-01-11 12:39:33 +00:00
}
2016-01-24 18:06:57 +00:00
hash = common . GetOne ( )
srcHash , err := src . Hash ( hash )
2014-03-28 17:56:04 +00:00
if err != nil {
Stats . Error ( )
2016-06-18 08:32:14 +00:00
ErrorLog ( src , "Failed to calculate src hash: %v" , err )
2016-01-24 18:06:57 +00:00
return false , hash , err
2015-08-20 19:48:58 +00:00
}
2016-01-11 12:39:33 +00:00
if srcHash == "" {
2016-01-24 18:06:57 +00:00
return true , HashNone , nil
2014-03-28 17:56:04 +00:00
}
2016-01-24 18:06:57 +00:00
dstHash , err := dst . Hash ( hash )
2014-03-28 17:56:04 +00:00
if err != nil {
Stats . Error ( )
2016-06-18 08:32:14 +00:00
ErrorLog ( dst , "Failed to calculate dst hash: %v" , err )
2016-01-24 18:06:57 +00:00
return false , hash , err
2015-08-20 19:48:58 +00:00
}
2016-01-11 12:39:33 +00:00
if dstHash == "" {
2016-01-24 18:06:57 +00:00
return true , HashNone , nil
2014-03-28 17:56:04 +00:00
}
2016-01-24 18:06:57 +00:00
return srcHash == dstHash , hash , nil
2014-03-28 17:56:04 +00:00
}
2015-09-22 17:47:16 +00:00
// Equal checks to see if the src and dst objects are equal by looking at
2016-01-11 12:39:33 +00:00
// size, mtime and hash
2014-03-28 17:56:04 +00:00
//
// If the src and dst size are different then it is considered to be
2015-06-06 07:38:45 +00:00
// not equal. If --size-only is in effect then this is the only check
2016-06-17 16:20:08 +00:00
// that is done. If --ignore-size is in effect then this check is
// skipped and the files are considered the same size.
2014-03-28 17:56:04 +00:00
//
// If the size is the same and the mtime is the same then it is
2015-06-06 07:38:45 +00:00
// considered to be equal. This check is skipped if using --checksum.
2014-03-28 17:56:04 +00:00
//
2015-06-06 07:38:45 +00:00
// If the size is the same and mtime is different, unreadable or
2016-01-11 12:39:33 +00:00
// --checksum is set and the hash is the same then the file is
2015-06-06 07:38:45 +00:00
// considered to be equal. In this case the mtime on the dst is
// updated if --checksum is not set.
2014-03-28 17:56:04 +00:00
//
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal ( src , dst Object ) bool {
2016-12-18 10:03:56 +00:00
return equal ( src , dst , Config . SizeOnly , Config . CheckSum )
}
func equal ( src , dst Object , sizeOnly , checkSum bool ) bool {
2016-06-17 16:20:08 +00:00
if ! Config . IgnoreSize {
if src . Size ( ) != dst . Size ( ) {
Debug ( src , "Sizes differ" )
return false
}
2014-03-28 17:56:04 +00:00
}
2016-12-18 10:03:56 +00:00
if sizeOnly {
2015-06-06 07:38:45 +00:00
Debug ( src , "Sizes identical" )
return true
}
2014-03-28 17:56:04 +00:00
2016-11-28 17:08:15 +00:00
// Assert: Size is equal or being ignored
// If checking checksum and not modtime
2016-12-18 10:03:56 +00:00
if checkSum {
2016-11-28 17:08:15 +00:00
// Check the hash
same , hash , _ := CheckHashes ( src , dst )
if ! same {
Debug ( src , "%v differ" , hash )
return false
2015-08-20 19:48:58 +00:00
}
2016-11-28 17:08:15 +00:00
if hash == HashNone {
Debug ( src , "Size of src and dst objects identical" )
2015-06-03 14:08:27 +00:00
} else {
2016-11-28 17:08:15 +00:00
Debug ( src , "Size and %v of src and dst objects identical" , hash )
2015-06-03 14:08:27 +00:00
}
2016-11-28 17:08:15 +00:00
return true
}
// Sizes the same so check the mtime
if Config . ModifyWindow == ModTimeNotSupported {
Debug ( src , "Sizes identical" )
return true
}
srcModTime := src . ModTime ( )
dstModTime := dst . ModTime ( )
dt := dstModTime . Sub ( srcModTime )
ModifyWindow := Config . ModifyWindow
if dt < ModifyWindow && dt > - ModifyWindow {
Debug ( src , "Size and modification time the same (differ by %s, within tolerance %s)" , dt , ModifyWindow )
return true
2014-03-28 17:56:04 +00:00
}
2016-11-28 17:08:15 +00:00
Debug ( src , "Modification times differ by %s: %v, %v" , dt , srcModTime , dstModTime )
// Check if the hashes are the same
2016-01-24 18:06:57 +00:00
same , hash , _ := CheckHashes ( src , dst )
2014-03-28 17:56:04 +00:00
if ! same {
2016-11-28 17:08:15 +00:00
Debug ( src , "%v differ" , hash )
return false
}
if hash == HashNone {
// if couldn't check hash, return that they differ
2014-03-28 17:56:04 +00:00
return false
}
2016-11-28 17:08:15 +00:00
// mod time differs but hash is the same to reset mod time if required
if ! Config . NoUpdateModTime {
2016-01-11 12:39:33 +00:00
// Size and hash the same but mtime different so update the
2015-06-03 14:08:27 +00:00
// mtime of the dst object here
2016-03-22 15:07:10 +00:00
err := dst . SetModTime ( srcModTime )
2016-03-22 15:23:37 +00:00
if err == ErrorCantSetModTime {
Debug ( src , "src and dst identical but can't set mod time without re-uploading" )
return false
} else if err != nil {
2016-03-22 15:07:10 +00:00
Stats . Error ( )
2016-11-28 17:08:15 +00:00
ErrorLog ( dst , "Failed to set modification time: %v" , err )
} else {
Debug ( src , "Updated modification time in destination" )
2016-03-22 15:07:10 +00:00
}
2015-06-03 14:08:27 +00:00
}
2014-03-28 17:56:04 +00:00
return true
}
2016-09-21 21:13:24 +00:00
// MimeTypeFromName returns a guess at the mime type from the name
func MimeTypeFromName ( remote string ) ( mimeType string ) {
mimeType = mime . TypeByExtension ( path . Ext ( remote ) )
2016-04-07 13:32:01 +00:00
if ! strings . ContainsRune ( mimeType , '/' ) {
2015-03-01 12:38:31 +00:00
mimeType = "application/octet-stream"
}
return mimeType
}
2016-09-21 21:13:24 +00:00
// MimeType returns the MimeType from the object, either by calling
// the MimeTyper interface or using MimeTypeFromName
func MimeType ( o ObjectInfo ) ( mimeType string ) {
// Read the MimeType from the optional interface if available
if do , ok := o . ( MimeTyper ) ; ok {
mimeType = do . MimeType ( )
Debug ( o , "Read MimeType as %q" , mimeType )
if mimeType != "" {
return mimeType
}
}
return MimeTypeFromName ( o . Remote ( ) )
}
2014-07-15 18:27:05 +00:00
// Used to remove a failed copy
2015-03-14 17:54:41 +00:00
//
// Returns whether the file was succesfully removed or not
func removeFailedCopy ( dst Object ) bool {
if dst == nil {
return false
}
Debug ( dst , "Removing failed copy" )
removeErr := dst . Remove ( )
if removeErr != nil {
Debug ( dst , "Failed to remove failed copy: %s" , removeErr )
return false
2014-07-15 18:27:05 +00:00
}
2015-03-14 17:54:41 +00:00
return true
2014-07-15 18:27:05 +00:00
}
2016-10-23 16:34:17 +00:00
// Wrapper to override the remote for an object
type overrideRemoteObject struct {
Object
remote string
}
// Remote returns the overriden remote name
func ( o * overrideRemoteObject ) Remote ( ) string {
return o . remote
}
2016-10-22 16:53:10 +00:00
// Copy src object to dst or f if nil. If dst is nil then it uses
// remote as the name of the new object.
func Copy ( f Fs , dst Object , remote string , src Object ) ( err error ) {
if Config . DryRun {
Log ( src , "Not copying as --dry-run" )
return nil
}
2016-01-12 17:38:28 +00:00
maxTries := Config . LowLevelRetries
2015-02-02 17:29:08 +00:00
tries := 0
doUpdate := dst != nil
2016-06-18 09:55:58 +00:00
var actionTaken string
for {
// Try server side copy first - if has optional interface and
// is same underlying remote
actionTaken = "Copied (server side copy)"
2017-01-13 17:21:47 +00:00
if doCopy := f . Features ( ) . Copy ; doCopy != nil && SameConfig ( src . Fs ( ) , f ) {
2016-06-18 09:55:58 +00:00
var newDst Object
2017-01-13 17:21:47 +00:00
newDst , err = doCopy ( src , remote )
2016-06-18 09:55:58 +00:00
if err == nil {
dst = newDst
}
} else {
err = ErrorCantCopy
2015-10-06 14:35:22 +00:00
}
2016-06-18 09:55:58 +00:00
// If can't server side copy, do it manually
if err == ErrorCantCopy {
var in0 io . ReadCloser
in0 , err = src . Open ( )
if err != nil {
err = errors . Wrap ( err , "failed to open source object" )
} else {
2016-12-14 21:15:12 +00:00
in := NewAccountWithBuffer ( in0 , src ) // account and buffer the transfer
2014-03-28 17:56:04 +00:00
2016-10-23 16:34:17 +00:00
wrappedSrc := & overrideRemoteObject { Object : src , remote : remote }
2016-06-18 09:55:58 +00:00
if doUpdate {
2016-07-09 09:11:57 +00:00
actionTaken = "Copied (replaced existing)"
2016-10-23 16:34:17 +00:00
err = dst . Update ( in , wrappedSrc )
2016-06-18 09:55:58 +00:00
} else {
actionTaken = "Copied (new)"
2016-10-23 16:34:17 +00:00
dst , err = f . Put ( in , wrappedSrc )
2016-06-18 09:55:58 +00:00
}
closeErr := in . Close ( )
if err == nil {
err = closeErr
}
}
2015-02-14 18:48:08 +00:00
}
2015-02-02 17:29:08 +00:00
tries ++
2016-06-18 09:55:58 +00:00
if tries >= maxTries {
break
2015-03-14 17:54:41 +00:00
}
2016-06-18 09:55:58 +00:00
// Retry if err returned a retry error
if IsRetryError ( err ) || ShouldRetry ( err ) {
2016-06-25 20:22:50 +00:00
Debug ( src , "Received error: %v - low level retry %d/%d" , err , tries , maxTries )
2016-06-18 09:55:58 +00:00
continue
}
// otherwise finish
break
2014-03-28 17:56:04 +00:00
}
if err != nil {
Stats . Error ( )
2016-06-18 08:32:14 +00:00
ErrorLog ( src , "Failed to copy: %v" , err )
2016-07-04 12:12:33 +00:00
return err
2014-03-28 17:56:04 +00:00
}
2014-07-15 18:27:05 +00:00
2014-07-19 11:38:58 +00:00
// Verify sizes are the same after transfer
2016-06-17 16:20:08 +00:00
if ! Config . IgnoreSize && src . Size ( ) != dst . Size ( ) {
2014-07-19 11:38:58 +00:00
Stats . Error ( )
2016-06-12 14:06:02 +00:00
err = errors . Errorf ( "corrupted on transfer: sizes differ %d vs %d" , src . Size ( ) , dst . Size ( ) )
2016-06-18 08:32:14 +00:00
ErrorLog ( dst , "%v" , err )
2014-07-19 11:38:58 +00:00
removeFailedCopy ( dst )
2016-07-04 12:12:33 +00:00
return err
2014-07-19 11:38:58 +00:00
}
2016-01-11 12:39:33 +00:00
// Verify hashes are the same after transfer - ignoring blank hashes
// TODO(klauspost): This could be extended, so we always create a hash type matching
// the destination, and calculate it while sending.
common := src . Fs ( ) . Hashes ( ) . Overlap ( dst . Fs ( ) . Hashes ( ) )
2016-01-24 18:06:57 +00:00
// Debug(src, "common hashes: %v", common)
2016-01-11 12:39:33 +00:00
if ! Config . SizeOnly && common . Count ( ) > 0 {
// Get common hash type
hashType := common . GetOne ( )
2016-07-04 12:12:33 +00:00
var srcSum string
srcSum , err = src . Hash ( hashType )
2016-01-11 12:39:33 +00:00
if err != nil {
2014-07-15 18:27:05 +00:00
Stats . Error ( )
2016-06-18 08:32:14 +00:00
ErrorLog ( src , "Failed to read src hash: %v" , err )
2016-01-11 12:39:33 +00:00
} else if srcSum != "" {
2016-07-04 12:12:33 +00:00
var dstSum string
dstSum , err = dst . Hash ( hashType )
2016-01-11 12:39:33 +00:00
if err != nil {
2015-06-09 12:18:40 +00:00
Stats . Error ( )
2016-06-18 08:32:14 +00:00
ErrorLog ( dst , "Failed to read hash: %v" , err )
2016-01-11 12:39:33 +00:00
} else if ! HashEquals ( srcSum , dstSum ) {
2015-06-09 12:18:40 +00:00
Stats . Error ( )
2016-06-12 14:06:02 +00:00
err = errors . Errorf ( "corrupted on transfer: %v hash differ %q vs %q" , hashType , srcSum , dstSum )
2016-06-18 08:32:14 +00:00
ErrorLog ( dst , "%v" , err )
2015-06-09 12:18:40 +00:00
removeFailedCopy ( dst )
2016-07-04 12:12:33 +00:00
return err
2015-06-09 12:18:40 +00:00
}
2014-07-15 18:27:05 +00:00
}
}
2014-04-18 16:46:57 +00:00
Debug ( src , actionTaken )
2016-07-04 12:12:33 +00:00
return err
2015-08-24 20:42:23 +00:00
}
2016-10-22 16:53:10 +00:00
// Move src object to dst or fdst if nil. If dst is nil then it uses
// remote as the name of the new object.
func Move ( fdst Fs , dst Object , remote string , src Object ) ( err error ) {
if Config . DryRun {
Log ( src , "Not moving as --dry-run" )
return nil
}
// See if we have Move available
2017-01-13 17:21:47 +00:00
if doMove := fdst . Features ( ) . Move ; doMove != nil && SameConfig ( src . Fs ( ) , fdst ) {
2016-10-22 16:53:10 +00:00
// Delete destination if it exists
if dst != nil {
err = DeleteFile ( dst )
if err != nil {
return err
}
}
// Move dst <- src
2017-01-13 17:21:47 +00:00
_ , err := doMove ( src , remote )
2016-10-22 16:53:10 +00:00
switch err {
case nil :
2016-12-09 16:50:28 +00:00
Debug ( src , "Moved (server side)" )
2016-10-22 16:53:10 +00:00
return nil
case ErrorCantMove :
Debug ( src , "Can't move, switching to copy" )
default :
Stats . Error ( )
ErrorLog ( dst , "Couldn't move: %v" , err )
return err
}
}
// Move not found or didn't work so copy dst <- src
err = Copy ( fdst , dst , remote , src )
if err != nil {
ErrorLog ( src , "Not deleting source as copy failed: %v" , err )
return err
}
// Delete src if no error on copy
return DeleteFile ( src )
}
2017-01-10 20:03:55 +00:00
// CanServerSideMove returns true if fdst support server side moves or
// server side copies
//
// Some remotes simulate rename by server-side copy and delete, so include
// remotes that implements either Mover or Copier.
func CanServerSideMove ( fdst Fs ) bool {
2017-01-13 17:21:47 +00:00
canMove := fdst . Features ( ) . Move != nil
canCopy := fdst . Features ( ) . Copy != nil
2017-01-10 20:03:55 +00:00
return canMove || canCopy
}
2017-01-10 21:47:03 +00:00
// deleteFileWithBackupDir deletes a single file respecting --dry-run
// and accumulating stats and errors.
//
// If backupDir is set then it moves the file to there instead of
// deleting
func deleteFileWithBackupDir ( dst Object , backupDir Fs ) ( err error ) {
Stats . Checking ( dst . Remote ( ) )
action , actioned , actioning := "delete" , "Deleted" , "deleting"
if backupDir != nil {
action , actioned , actioning = "move into backup dir" , "Moved into backup dir" , "moving into backup dir"
}
2016-03-05 16:10:51 +00:00
if Config . DryRun {
2017-01-10 21:47:03 +00:00
Log ( dst , "Not %s as --dry-run" , actioning )
} else if backupDir != nil {
if ! SameConfig ( dst . Fs ( ) , backupDir ) {
err = errors . New ( "parameter to --backup-dir has to be on the same remote as destination" )
2016-03-05 16:10:51 +00:00
} else {
2017-01-19 17:26:29 +00:00
remoteWithSuffix := dst . Remote ( ) + Config . Suffix
overwritten , _ := backupDir . NewObject ( remoteWithSuffix )
err = Move ( backupDir , overwritten , remoteWithSuffix , dst )
2016-03-05 16:10:51 +00:00
}
2017-01-10 21:47:03 +00:00
} else {
err = dst . Remove ( )
}
if err != nil {
Stats . Error ( )
ErrorLog ( dst , "Couldn't %s: %v" , action , err )
} else {
Debug ( dst , actioned )
2016-03-05 16:10:51 +00:00
}
2017-01-10 21:47:03 +00:00
Stats . DoneChecking ( dst . Remote ( ) )
2016-06-25 13:27:44 +00:00
return err
2016-03-05 16:10:51 +00:00
}
2017-01-10 21:47:03 +00:00
// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
//
// If useBackupDir is set and --backup-dir is in effect then it moves
// the file to there instead of deleting
func DeleteFile ( dst Object ) ( err error ) {
return deleteFileWithBackupDir ( dst , nil )
}
// deleteFilesWithBackupDir removes all the files passed in the
// channel
//
// If backupDir is set the files will be placed into that directory
// instead of being deleted.
func deleteFilesWithBackupDir ( toBeDeleted ObjectsChan , backupDir Fs ) error {
2014-03-28 17:56:04 +00:00
var wg sync . WaitGroup
wg . Add ( Config . Transfers )
2016-06-25 13:27:44 +00:00
var errorCount int32
2014-03-28 17:56:04 +00:00
for i := 0 ; i < Config . Transfers ; i ++ {
go func ( ) {
defer wg . Done ( )
2015-09-22 17:47:16 +00:00
for dst := range toBeDeleted {
2017-01-10 21:47:03 +00:00
err := deleteFileWithBackupDir ( dst , backupDir )
2016-06-25 13:27:44 +00:00
if err != nil {
atomic . AddInt32 ( & errorCount , 1 )
}
2014-03-28 17:56:04 +00:00
}
} ( )
}
2014-07-22 22:03:14 +00:00
Log ( nil , "Waiting for deletions to finish" )
2014-03-28 17:56:04 +00:00
wg . Wait ( )
2016-06-25 13:27:44 +00:00
if errorCount > 0 {
return errors . Errorf ( "failed to delete %d files" , errorCount )
}
return nil
2014-03-28 17:56:04 +00:00
}
2017-01-10 21:47:03 +00:00
// DeleteFiles removes all the files passed in the channel
func DeleteFiles ( toBeDeleted ObjectsChan ) error {
return deleteFilesWithBackupDir ( toBeDeleted , nil )
}
2016-06-25 13:28:26 +00:00
// Read a Objects into add() for the given Fs.
2016-04-23 20:46:52 +00:00
// dir is the start directory, "" for root
2016-01-12 13:33:03 +00:00
// If includeAll is specified all files will be added,
// otherwise only files passing the filter will be added.
2016-07-04 12:12:33 +00:00
//
// Each object is passed ito the function provided. If that returns
// an error then the listing will be aborted and that error returned.
func readFilesFn ( fs Fs , includeAll bool , dir string , add func ( Object ) error ) ( err error ) {
2016-04-21 19:06:21 +00:00
list := NewLister ( )
if ! includeAll {
list . SetFilter ( Config . Filter )
2016-06-02 20:02:44 +00:00
list . SetLevel ( Config . MaxDepth )
2016-04-21 19:06:21 +00:00
}
2016-04-23 20:46:52 +00:00
list . Start ( fs , dir )
2016-04-21 19:06:21 +00:00
for {
o , err := list . GetObject ( )
if err != nil {
2016-06-25 13:28:26 +00:00
return err
2016-04-21 19:06:21 +00:00
}
// Check if we are finished
if o == nil {
break
}
2016-06-25 13:28:26 +00:00
// Make sure we don't delete excluded files if not required
if includeAll || Config . Filter . IncludeObject ( o ) {
2016-07-04 12:12:33 +00:00
err = add ( o )
if err != nil {
list . SetError ( err )
}
2016-06-25 13:28:26 +00:00
} else {
Debug ( o , "Excluded from sync (and deletion)" )
}
}
return nil
}
// Read a map of Object.Remote to Object for the given Fs.
// dir is the start directory, "" for root
// If includeAll is specified all files will be added,
// otherwise only files passing the filter will be added.
//
// This also detects duplicates and normalised duplicates
func readFilesMap ( fs Fs , includeAll bool , dir string ) ( files map [ string ] Object , err error ) {
files = make ( map [ string ] Object )
normalised := make ( map [ string ] struct { } )
2016-07-04 12:12:33 +00:00
err = readFilesFn ( fs , includeAll , dir , func ( o Object ) error {
2015-03-14 17:11:24 +00:00
remote := o . Remote ( )
2016-01-23 20:16:47 +00:00
normalisedRemote := strings . ToLower ( norm . NFC . String ( remote ) )
2015-03-14 17:11:24 +00:00
if _ , ok := files [ remote ] ; ! ok {
2016-06-25 13:28:26 +00:00
files [ remote ] = o
if _ , ok := normalised [ normalisedRemote ] ; ok {
Log ( o , "Warning: File found with same name but different case on %v" , o . Fs ( ) )
2015-11-12 11:46:04 +00:00
}
2015-03-14 17:11:24 +00:00
} else {
Log ( o , "Duplicate file detected" )
}
2016-01-23 20:16:47 +00:00
normalised [ normalisedRemote ] = struct { } { }
2016-07-04 12:12:33 +00:00
return nil
2016-06-25 13:28:26 +00:00
} )
2016-11-05 18:03:55 +00:00
if err != nil {
err = errors . Wrapf ( err , "error listing: %s" , fs )
}
2016-06-25 13:28:26 +00:00
return files , err
2016-04-21 19:06:21 +00:00
}
// readFilesMaps runs readFilesMap on fdst and fsrc at the same time
2016-04-23 20:46:52 +00:00
// dir is the start directory, "" for root
func readFilesMaps ( fdst Fs , fdstIncludeAll bool , fsrc Fs , fsrcIncludeAll bool , dir string ) ( dstFiles , srcFiles map [ string ] Object , err error ) {
2016-04-21 19:06:21 +00:00
var wg sync . WaitGroup
var srcErr , dstErr error
list := func ( fs Fs , includeAll bool , pMap * map [ string ] Object , pErr * error ) {
defer wg . Done ( )
Log ( fs , "Building file list" )
2016-06-21 20:17:52 +00:00
files , listErr := readFilesMap ( fs , includeAll , dir )
2016-04-21 19:06:21 +00:00
if listErr != nil {
ErrorLog ( fs , "Error building file list: %v" , listErr )
* pErr = listErr
} else {
Debug ( fs , "Done building file list" )
2016-06-21 20:17:52 +00:00
* pMap = files
2016-04-21 19:06:21 +00:00
}
}
wg . Add ( 2 )
go list ( fdst , fdstIncludeAll , & dstFiles , & srcErr )
go list ( fsrc , fsrcIncludeAll , & srcFiles , & dstErr )
wg . Wait ( )
if srcErr != nil {
err = srcErr
}
if dstErr != nil {
err = dstErr
}
return dstFiles , srcFiles , err
2015-03-14 17:11:24 +00:00
}
2017-01-11 14:59:53 +00:00
// SameConfig returns true if fdst and fsrc are using the same config
// file entry
func SameConfig ( fdst , fsrc Info ) bool {
return fdst . Name ( ) == fsrc . Name ( )
}
2015-09-22 17:47:16 +00:00
// Same returns true if fdst and fsrc point to the same underlying Fs
2017-01-11 14:59:53 +00:00
func Same ( fdst , fsrc Info ) bool {
return SameConfig ( fdst , fsrc ) && fdst . Root ( ) == fsrc . Root ( )
2015-09-01 19:50:28 +00:00
}
2016-07-11 10:36:46 +00:00
// Overlapping returns true if fdst and fsrc point to the same
2017-01-11 14:59:53 +00:00
// underlying Fs and they overlap.
func Overlapping ( fdst , fsrc Info ) bool {
if ! SameConfig ( fdst , fsrc ) {
return false
}
// Return the Root with a trailing / if not empty
fixedRoot := func ( f Info ) string {
s := strings . Trim ( f . Root ( ) , "/" )
if s != "" {
s += "/"
}
return s
}
fdstRoot := fixedRoot ( fdst )
fsrcRoot := fixedRoot ( fsrc )
return strings . HasPrefix ( fdstRoot , fsrcRoot ) || strings . HasPrefix ( fsrcRoot , fdstRoot )
2016-07-11 10:36:46 +00:00
}
2016-04-07 13:56:27 +00:00
// checkIdentical checks to see if dst and src are identical
//
// it returns true if differences were found
2016-10-12 09:59:55 +00:00
// it also returns whether it couldn't be hashed
2016-11-05 18:17:21 +00:00
func checkIdentical ( dst , src Object ) ( differ bool , noHash bool ) {
2016-07-02 15:58:50 +00:00
Stats . Checking ( src . Remote ( ) )
defer Stats . DoneChecking ( src . Remote ( ) )
2016-04-07 13:56:27 +00:00
if src . Size ( ) != dst . Size ( ) {
Stats . Error ( )
ErrorLog ( src , "Sizes differ" )
2016-10-12 09:59:55 +00:00
return true , false
2016-04-07 13:56:27 +00:00
}
if ! Config . SizeOnly {
2016-10-12 09:59:55 +00:00
same , hash , err := CheckHashes ( src , dst )
2016-04-07 13:56:27 +00:00
if err != nil {
// CheckHashes will log and count errors
2016-10-12 09:59:55 +00:00
return true , false
}
if hash == HashNone {
2016-11-05 18:17:21 +00:00
return false , true
2016-04-07 13:56:27 +00:00
}
if ! same {
Stats . Error ( )
2016-10-12 09:59:55 +00:00
ErrorLog ( src , "%v differ" , hash )
return true , false
2016-04-07 13:56:27 +00:00
}
}
Debug ( src , "OK" )
2016-10-12 09:59:55 +00:00
return false , false
2016-04-07 13:56:27 +00:00
}
2016-01-11 12:39:33 +00:00
// Check the files in fsrc and fdst according to Size and hash
2014-03-28 17:56:04 +00:00
func Check ( fdst , fsrc Fs ) error {
2016-04-23 20:46:52 +00:00
dstFiles , srcFiles , err := readFilesMaps ( fdst , false , fsrc , false , "" )
2016-04-21 19:06:21 +00:00
if err != nil {
return err
}
2016-01-17 10:08:28 +00:00
differences := int32 ( 0 )
2016-10-12 09:59:55 +00:00
noHashes := int32 ( 0 )
2015-11-24 16:54:12 +00:00
// FIXME could do this as it goes along and make it use less
// memory.
2015-03-14 17:11:24 +00:00
// Move all the common files into commonFiles and delete then
// from srcFiles and dstFiles
2014-03-28 17:56:04 +00:00
commonFiles := make ( map [ string ] [ ] Object )
2015-03-14 17:11:24 +00:00
for remote , src := range srcFiles {
2014-03-28 17:56:04 +00:00
if dst , ok := dstFiles [ remote ] ; ok {
commonFiles [ remote ] = [ ] Object { dst , src }
2015-03-14 17:11:24 +00:00
delete ( srcFiles , remote )
2014-03-28 17:56:04 +00:00
delete ( dstFiles , remote )
}
}
Log ( fdst , "%d files not in %v" , len ( dstFiles ) , fsrc )
for _ , dst := range dstFiles {
Stats . Error ( )
2015-08-08 19:10:31 +00:00
ErrorLog ( dst , "File not in %v" , fsrc )
2016-01-17 10:08:28 +00:00
atomic . AddInt32 ( & differences , 1 )
2014-03-28 17:56:04 +00:00
}
Log ( fsrc , "%d files not in %s" , len ( srcFiles ) , fdst )
for _ , src := range srcFiles {
Stats . Error ( )
2015-08-08 19:10:31 +00:00
ErrorLog ( src , "File not in %v" , fdst )
2016-01-17 10:08:28 +00:00
atomic . AddInt32 ( & differences , 1 )
2014-03-28 17:56:04 +00:00
}
checks := make ( chan [ ] Object , Config . Transfers )
go func ( ) {
for _ , check := range commonFiles {
checks <- check
}
close ( checks )
} ( )
var checkerWg sync . WaitGroup
checkerWg . Add ( Config . Checkers )
for i := 0 ; i < Config . Checkers ; i ++ {
go func ( ) {
defer checkerWg . Done ( )
for check := range checks {
2016-10-12 09:59:55 +00:00
differ , noHash := checkIdentical ( check [ 0 ] , check [ 1 ] )
if differ {
2016-01-17 10:08:28 +00:00
atomic . AddInt32 ( & differences , 1 )
2014-03-28 17:56:04 +00:00
}
2016-10-12 09:59:55 +00:00
if noHash {
atomic . AddInt32 ( & noHashes , 1 )
}
2014-03-28 17:56:04 +00:00
}
} ( )
}
Log ( fdst , "Waiting for checks to finish" )
checkerWg . Wait ( )
Log ( fdst , "%d differences found" , Stats . GetErrors ( ) )
2016-10-12 09:59:55 +00:00
if noHashes > 0 {
Log ( fdst , "%d hashes could not be checked" , noHashes )
}
2016-01-17 10:08:28 +00:00
if differences > 0 {
2016-06-12 14:06:02 +00:00
return errors . Errorf ( "%d differences found" , differences )
2014-03-28 17:56:04 +00:00
}
return nil
}
2015-09-22 17:47:16 +00:00
// ListFn lists the Fs to the supplied function
2014-03-28 17:56:04 +00:00
//
// Lists in parallel which may get them out of order
2014-07-12 11:09:20 +00:00
func ListFn ( f Fs , fn func ( Object ) ) error {
2016-06-02 20:02:44 +00:00
list := NewLister ( ) . SetFilter ( Config . Filter ) . SetLevel ( Config . MaxDepth ) . Start ( f , "" )
2014-03-28 17:56:04 +00:00
var wg sync . WaitGroup
wg . Add ( Config . Checkers )
for i := 0 ; i < Config . Checkers ; i ++ {
go func ( ) {
defer wg . Done ( )
2016-04-21 19:06:21 +00:00
for {
o , err := list . GetObject ( )
if err != nil {
2017-01-10 04:14:53 +00:00
// The error will be persisted within the Lister object and
// we'll get an opportunity to return it as we leave this
// function.
return
2016-04-21 19:06:21 +00:00
}
// check if we are finished
if o == nil {
return
}
2015-11-24 16:54:12 +00:00
if Config . Filter . IncludeObject ( o ) {
fn ( o )
}
2014-03-28 17:56:04 +00:00
}
} ( )
}
wg . Wait ( )
2017-01-10 04:14:53 +00:00
return list . Error ( )
2014-03-28 17:56:04 +00:00
}
2015-02-28 15:30:40 +00:00
// mutex for synchronized output
var outMutex sync . Mutex
// Synchronized fmt.Fprintf
2015-09-22 06:31:12 +00:00
//
// Ignores errors from Fprintf
func syncFprintf ( w io . Writer , format string , a ... interface { } ) {
2015-02-28 15:30:40 +00:00
outMutex . Lock ( )
defer outMutex . Unlock ( )
2015-09-22 06:31:12 +00:00
_ , _ = fmt . Fprintf ( w , format , a ... )
2015-02-28 15:30:40 +00:00
}
2015-09-15 14:46:06 +00:00
// List the Fs to the supplied writer
2014-07-12 11:09:20 +00:00
//
2015-11-24 16:54:12 +00:00
// Shows size and path - obeys includes and excludes
2014-07-12 11:09:20 +00:00
//
// Lists in parallel which may get them out of order
2014-08-01 16:58:39 +00:00
func List ( f Fs , w io . Writer ) error {
2014-07-12 11:09:20 +00:00
return ListFn ( f , func ( o Object ) {
2015-02-28 15:30:40 +00:00
syncFprintf ( w , "%9d %s\n" , o . Size ( ) , o . Remote ( ) )
2014-07-12 11:09:20 +00:00
} )
}
2015-09-22 17:47:16 +00:00
// ListLong lists the Fs to the supplied writer
2014-07-12 11:09:20 +00:00
//
2015-11-24 16:54:12 +00:00
// Shows size, mod time and path - obeys includes and excludes
2014-07-12 11:09:20 +00:00
//
// Lists in parallel which may get them out of order
2014-08-01 16:58:39 +00:00
func ListLong ( f Fs , w io . Writer ) error {
2014-07-12 11:09:20 +00:00
return ListFn ( f , func ( o Object ) {
2016-07-02 15:58:50 +00:00
Stats . Checking ( o . Remote ( ) )
2014-07-12 11:09:20 +00:00
modTime := o . ModTime ( )
2016-07-02 15:58:50 +00:00
Stats . DoneChecking ( o . Remote ( ) )
2015-09-22 18:04:12 +00:00
syncFprintf ( w , "%9d %s %s\n" , o . Size ( ) , modTime . Local ( ) . Format ( "2006-01-02 15:04:05.000000000" ) , o . Remote ( ) )
2014-07-12 11:09:20 +00:00
} )
}
2015-09-22 17:47:16 +00:00
// Md5sum list the Fs to the supplied writer
2014-07-12 11:09:20 +00:00
//
2015-11-24 16:54:12 +00:00
// Produces the same output as the md5sum command - obeys includes and
// excludes
2014-07-12 11:09:20 +00:00
//
// Lists in parallel which may get them out of order
2014-08-01 16:58:39 +00:00
func Md5sum ( f Fs , w io . Writer ) error {
2016-01-11 12:39:33 +00:00
return hashLister ( HashMD5 , f , w )
}
// Sha1sum list the Fs to the supplied writer
//
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func Sha1sum ( f Fs , w io . Writer ) error {
return hashLister ( HashSHA1 , f , w )
}
func hashLister ( ht HashType , f Fs , w io . Writer ) error {
2014-07-12 11:09:20 +00:00
return ListFn ( f , func ( o Object ) {
2016-07-02 15:58:50 +00:00
Stats . Checking ( o . Remote ( ) )
2016-01-11 12:39:33 +00:00
sum , err := o . Hash ( ht )
2016-07-02 15:58:50 +00:00
Stats . DoneChecking ( o . Remote ( ) )
2016-01-11 12:39:33 +00:00
if err == ErrHashUnsupported {
sum = "UNSUPPORTED"
} else if err != nil {
Debug ( o , "Failed to read %v: %v" , ht , err )
sum = "ERROR"
2014-07-12 11:09:20 +00:00
}
2016-01-17 13:56:00 +00:00
syncFprintf ( w , "%*s %s\n" , HashWidth [ ht ] , sum , o . Remote ( ) )
2014-07-12 11:09:20 +00:00
} )
}
2015-10-02 18:48:48 +00:00
// Count counts the objects and their sizes in the Fs
2015-11-24 16:54:12 +00:00
//
// Obeys includes and excludes
2015-10-02 18:48:48 +00:00
func Count ( f Fs ) ( objects int64 , size int64 , err error ) {
err = ListFn ( f , func ( o Object ) {
atomic . AddInt64 ( & objects , 1 )
atomic . AddInt64 ( & size , o . Size ( ) )
} )
return
}
2015-09-22 17:47:16 +00:00
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
2014-08-01 16:58:39 +00:00
func ListDir ( f Fs , w io . Writer ) error {
2016-06-02 20:02:44 +00:00
level := 1
if Config . MaxDepth > 0 {
level = Config . MaxDepth
}
2016-12-07 11:16:36 +00:00
list := NewLister ( ) . SetFilter ( Config . Filter ) . SetLevel ( level ) . Start ( f , "" )
2016-04-21 19:06:21 +00:00
for {
dir , err := list . GetDir ( )
if err != nil {
log . Fatal ( err )
}
if dir == nil {
break
}
2015-02-28 15:30:40 +00:00
syncFprintf ( w , "%12d %13s %9d %s\n" , dir . Bytes , dir . When . Format ( "2006-01-02 15:04:05" ) , dir . Count , dir . Name )
2014-03-28 17:56:04 +00:00
}
return nil
}
2015-09-22 17:47:16 +00:00
// Mkdir makes a destination directory or container
2016-11-25 21:52:43 +00:00
func Mkdir ( f Fs , dir string ) error {
2016-02-28 19:47:22 +00:00
if Config . DryRun {
Log ( f , "Not making directory as dry run is set" )
return nil
}
2016-11-25 21:52:43 +00:00
err := f . Mkdir ( dir )
2014-03-28 17:56:04 +00:00
if err != nil {
Stats . Error ( )
return err
}
return nil
}
2016-02-25 20:05:34 +00:00
// TryRmdir removes a container but not if not empty. It doesn't
// count errors but may return one.
2016-11-25 21:52:43 +00:00
func TryRmdir ( f Fs , dir string ) error {
2014-03-28 17:56:04 +00:00
if Config . DryRun {
2016-11-27 11:49:31 +00:00
if dir != "" {
Log ( dir , "Not deleting as dry run is set" )
} else {
Log ( f , "Not deleting as dry run is set" )
}
2016-02-25 20:05:34 +00:00
return nil
2014-03-28 17:56:04 +00:00
}
2016-11-25 21:52:43 +00:00
return f . Rmdir ( dir )
2016-02-25 20:05:34 +00:00
}
// Rmdir removes a container but not if not empty
2016-11-25 21:52:43 +00:00
func Rmdir ( f Fs , dir string ) error {
err := TryRmdir ( f , dir )
2016-02-25 20:05:34 +00:00
if err != nil {
Stats . Error ( )
return err
}
return err
2014-03-28 17:56:04 +00:00
}
2015-09-22 17:47:16 +00:00
// Purge removes a container and all of its contents
2014-03-28 17:56:04 +00:00
//
// FIXME doesn't delete local directories
func Purge ( f Fs ) error {
2015-11-08 14:16:00 +00:00
doFallbackPurge := true
2014-07-25 17:19:49 +00:00
var err error
2017-01-13 17:21:47 +00:00
if doPurge := f . Features ( ) . Purge ; doPurge != nil {
2015-11-08 14:16:00 +00:00
doFallbackPurge = false
2014-07-13 09:45:13 +00:00
if Config . DryRun {
2016-01-31 15:53:09 +00:00
Log ( f , "Not purging as --dry-run set" )
2014-07-13 09:45:13 +00:00
} else {
2017-01-13 17:21:47 +00:00
err = doPurge ( )
2015-11-08 14:16:00 +00:00
if err == ErrorCantPurge {
doFallbackPurge = true
}
2014-03-28 17:56:04 +00:00
}
2015-11-08 14:16:00 +00:00
}
if doFallbackPurge {
2014-07-25 17:19:49 +00:00
// DeleteFiles and Rmdir observe --dry-run
2016-04-23 20:46:52 +00:00
list := NewLister ( ) . Start ( f , "" )
2016-06-25 13:27:44 +00:00
err = DeleteFiles ( listToChan ( list ) )
if err != nil {
return err
}
2016-11-25 21:52:43 +00:00
err = Rmdir ( f , "" )
2014-07-25 17:19:49 +00:00
}
if err != nil {
Stats . Error ( )
return err
2014-03-28 17:56:04 +00:00
}
return nil
}
2015-12-02 22:25:32 +00:00
// Delete removes all the contents of a container. Unlike Purge, it
// obeys includes and excludes.
func Delete ( f Fs ) error {
delete := make ( ObjectsChan , Config . Transfers )
2016-06-25 13:27:44 +00:00
delErr := make ( chan error , 1 )
2015-12-02 22:25:32 +00:00
go func ( ) {
2016-06-25 13:27:44 +00:00
delErr <- DeleteFiles ( delete )
2015-12-02 22:25:32 +00:00
} ( )
err := ListFn ( f , func ( o Object ) {
delete <- o
} )
close ( delete )
2016-06-25 13:27:44 +00:00
delError := <- delErr
if err == nil {
err = delError
}
2015-12-02 22:25:32 +00:00
return err
}
2016-01-31 12:58:41 +00:00
2016-03-05 16:10:51 +00:00
// dedupeRename renames the objs slice to different names
func dedupeRename ( remote string , objs [ ] Object ) {
f := objs [ 0 ] . Fs ( )
2017-01-13 17:21:47 +00:00
doMove := f . Features ( ) . Move
if doMove == nil {
2016-03-05 16:10:51 +00:00
log . Fatalf ( "Fs %v doesn't support Move" , f )
}
ext := path . Ext ( remote )
base := remote [ : len ( remote ) - len ( ext ) ]
for i , o := range objs {
newName := fmt . Sprintf ( "%s-%d%s" , base , i + 1 , ext )
if ! Config . DryRun {
2017-01-13 17:21:47 +00:00
newObj , err := doMove ( o , newName )
2016-03-05 16:10:51 +00:00
if err != nil {
Stats . Error ( )
ErrorLog ( o , "Failed to rename: %v" , err )
continue
}
Log ( newObj , "renamed from: %v" , o )
} else {
Log ( remote , "Not renaming to %q as --dry-run" , newName )
}
}
}
// dedupeDeleteAllButOne deletes all but the one in keep
func dedupeDeleteAllButOne ( keep int , remote string , objs [ ] Object ) {
for i , o := range objs {
if i == keep {
continue
}
2016-06-25 13:27:44 +00:00
_ = DeleteFile ( o )
2016-03-05 16:10:51 +00:00
}
Log ( remote , "Deleted %d extra copies" , len ( objs ) - 1 )
}
// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
func dedupeDeleteIdentical ( remote string , objs [ ] Object ) [ ] Object {
// See how many of these duplicates are identical
byHash := make ( map [ string ] [ ] Object , len ( objs ) )
for _ , o := range objs {
md5sum , err := o . Hash ( HashMD5 )
if err == nil {
byHash [ md5sum ] = append ( byHash [ md5sum ] , o )
}
2016-01-31 12:58:41 +00:00
}
2016-03-05 16:10:51 +00:00
// Delete identical duplicates, refilling obj with the ones remaining
objs = nil
for md5sum , hashObjs := range byHash {
if len ( hashObjs ) > 1 {
Log ( remote , "Deleting %d/%d identical duplicates (md5sum %q)" , len ( hashObjs ) - 1 , len ( hashObjs ) , md5sum )
for _ , o := range hashObjs [ 1 : ] {
2016-06-25 13:27:44 +00:00
_ = DeleteFile ( o )
2016-03-05 16:10:51 +00:00
}
}
objs = append ( objs , hashObjs [ 0 ] )
}
return objs
}
// dedupeInteractive interactively dedupes the slice of objects
func dedupeInteractive ( remote string , objs [ ] Object ) {
fmt . Printf ( "%s: %d duplicates remain\n" , remote , len ( objs ) )
for i , o := range objs {
md5sum , err := o . Hash ( HashMD5 )
if err != nil {
md5sum = err . Error ( )
}
fmt . Printf ( " %d: %12d bytes, %s, md5sum %32s\n" , i + 1 , o . Size ( ) , o . ModTime ( ) . Format ( "2006-01-02 15:04:05.000000000" ) , md5sum )
}
switch Command ( [ ] string { "sSkip and do nothing" , "kKeep just one (choose which in next step)" , "rRename all to be different (by changing file.jpg to file-1.jpg)" } ) {
case 's' :
case 'k' :
keep := ChooseNumber ( "Enter the number of the file to keep" , 1 , len ( objs ) )
dedupeDeleteAllButOne ( keep - 1 , remote , objs )
case 'r' :
dedupeRename ( remote , objs )
}
}
type objectsSortedByModTime [ ] Object
func ( objs objectsSortedByModTime ) Len ( ) int { return len ( objs ) }
func ( objs objectsSortedByModTime ) Swap ( i , j int ) { objs [ i ] , objs [ j ] = objs [ j ] , objs [ i ] }
func ( objs objectsSortedByModTime ) Less ( i , j int ) bool {
return objs [ i ] . ModTime ( ) . Before ( objs [ j ] . ModTime ( ) )
}
// DeduplicateMode is how the dedupe command chooses what to do
type DeduplicateMode int
// Deduplicate modes
const (
DeduplicateInteractive DeduplicateMode = iota // interactively ask the user
DeduplicateSkip // skip all conflicts
DeduplicateFirst // choose the first object
DeduplicateNewest // choose the newest object
DeduplicateOldest // choose the oldest object
DeduplicateRename // rename the objects
)
2016-08-03 16:35:29 +00:00
func ( x DeduplicateMode ) String ( ) string {
switch x {
2016-03-05 16:10:51 +00:00
case DeduplicateInteractive :
return "interactive"
case DeduplicateSkip :
return "skip"
case DeduplicateFirst :
return "first"
case DeduplicateNewest :
return "newest"
case DeduplicateOldest :
return "oldest"
case DeduplicateRename :
return "rename"
}
return "unknown"
}
2016-08-03 16:35:29 +00:00
// Set a DeduplicateMode from a string
func ( x * DeduplicateMode ) Set ( s string ) error {
switch strings . ToLower ( s ) {
case "interactive" :
* x = DeduplicateInteractive
case "skip" :
* x = DeduplicateSkip
case "first" :
* x = DeduplicateFirst
case "newest" :
* x = DeduplicateNewest
case "oldest" :
* x = DeduplicateOldest
case "rename" :
* x = DeduplicateRename
default :
return errors . Errorf ( "Unknown mode for dedupe %q." , s )
}
return nil
}
// Type of the value
func ( x * DeduplicateMode ) Type ( ) string {
return "string"
}
// Check it satisfies the interface
var _ pflag . Value = ( * DeduplicateMode ) ( nil )
2016-03-05 16:10:51 +00:00
// Deduplicate interactively finds duplicate files and offers to
// delete all but one or rename them to be different. Only useful with
// Google Drive which can have duplicate file names.
func Deduplicate ( f Fs , mode DeduplicateMode ) error {
Log ( f , "Looking for duplicates using %v mode." , mode )
2016-01-31 12:58:41 +00:00
files := map [ string ] [ ] Object { }
2016-04-23 20:46:52 +00:00
list := NewLister ( ) . Start ( f , "" )
2016-04-21 19:06:21 +00:00
for {
o , err := list . GetObject ( )
if err != nil {
return err
}
// Check if we are finished
if o == nil {
break
}
2016-01-31 12:58:41 +00:00
remote := o . Remote ( )
files [ remote ] = append ( files [ remote ] , o )
}
for remote , objs := range files {
if len ( objs ) > 1 {
2016-03-05 16:10:51 +00:00
Log ( remote , "Found %d duplicates - deleting identical copies" , len ( objs ) )
objs = dedupeDeleteIdentical ( remote , objs )
if len ( objs ) <= 1 {
Log ( remote , "All duplicates removed" )
continue
2016-01-31 12:58:41 +00:00
}
2016-03-05 16:10:51 +00:00
switch mode {
case DeduplicateInteractive :
dedupeInteractive ( remote , objs )
case DeduplicateFirst :
dedupeDeleteAllButOne ( 0 , remote , objs )
case DeduplicateNewest :
sort . Sort ( objectsSortedByModTime ( objs ) ) // sort oldest first
dedupeDeleteAllButOne ( len ( objs ) - 1 , remote , objs )
case DeduplicateOldest :
sort . Sort ( objectsSortedByModTime ( objs ) ) // sort oldest first
dedupeDeleteAllButOne ( 0 , remote , objs )
case DeduplicateRename :
dedupeRename ( remote , objs )
case DeduplicateSkip :
// skip
default :
//skip
2016-01-31 12:58:41 +00:00
}
}
}
return nil
}
2016-04-21 19:06:21 +00:00
// listToChan will transfer all incoming objects to a new channel.
//
// If an error occurs, the error will be logged, and it will close the
// channel.
//
// If the error was ErrorDirNotFound then it will be ignored
func listToChan ( list * Lister ) ObjectsChan {
o := make ( ObjectsChan , Config . Checkers )
go func ( ) {
defer close ( o )
for {
obj , dir , err := list . Get ( )
if err != nil {
if err != ErrorDirNotFound {
Stats . Error ( )
ErrorLog ( nil , "Failed to list: %v" , err )
}
return
}
if dir == nil && obj == nil {
return
}
2016-07-25 18:13:20 +00:00
if obj == nil {
2016-04-21 19:06:21 +00:00
continue
}
o <- obj
}
} ( )
return o
}
2016-07-01 15:35:36 +00:00
// CleanUp removes the trash for the Fs
func CleanUp ( f Fs ) error {
2017-01-13 17:21:47 +00:00
doCleanUp := f . Features ( ) . CleanUp
if doCleanUp == nil {
2016-07-01 15:35:36 +00:00
return errors . Errorf ( "%v doesn't support cleanup" , f )
}
2016-07-02 15:58:50 +00:00
if Config . DryRun {
Log ( f , "Not running cleanup as --dry-run set" )
return nil
}
2017-01-13 17:21:47 +00:00
return doCleanUp ( )
2016-07-01 15:35:36 +00:00
}
2016-08-18 21:43:02 +00:00
// Cat any files to the io.Writer
func Cat ( f Fs , w io . Writer ) error {
var mu sync . Mutex
return ListFn ( f , func ( o Object ) {
2016-09-12 17:15:58 +00:00
var err error
2016-08-18 21:43:02 +00:00
Stats . Transferring ( o . Remote ( ) )
2016-09-12 17:15:58 +00:00
defer func ( ) {
Stats . DoneTransferring ( o . Remote ( ) , err == nil )
} ( )
2016-08-18 21:43:02 +00:00
mu . Lock ( )
defer mu . Unlock ( )
in , err := o . Open ( )
if err != nil {
Stats . Error ( )
ErrorLog ( o , "Failed to open: %v" , err )
return
}
defer func ( ) {
err = in . Close ( )
if err != nil {
Stats . Error ( )
ErrorLog ( o , "Failed to close: %v" , err )
}
} ( )
2016-12-14 21:15:12 +00:00
inAccounted := NewAccountWithBuffer ( in , o ) // account and buffer the transfer
2016-08-18 21:43:02 +00:00
_ , err = io . Copy ( w , inAccounted )
if err != nil {
Stats . Error ( )
ErrorLog ( o , "Failed to send to output: %v" , err )
}
} )
}
2016-11-27 11:49:31 +00:00
// Rmdirs removes any empty directories (or directories only
// containing empty directories) under f, including f.
func Rmdirs ( f Fs ) error {
list := NewLister ( ) . Start ( f , "" )
dirEmpty := make ( map [ string ] bool )
dirEmpty [ "" ] = true
for {
o , dir , err := list . Get ( )
if err != nil {
Stats . Error ( )
ErrorLog ( f , "Failed to list: %v" , err )
return err
} else if dir != nil {
// add a new directory as empty
dir := dir . Name
_ , found := dirEmpty [ dir ]
if ! found {
dirEmpty [ dir ] = true
}
} else if o != nil {
// mark the parents of the file as being non-empty
dir := o . Remote ( )
for dir != "" {
dir = path . Dir ( dir )
if dir == "." || dir == "/" {
dir = ""
}
empty , found := dirEmpty [ dir ]
// End if we reach a directory which is non-empty
if found && ! empty {
break
}
dirEmpty [ dir ] = false
}
} else {
// finished as dir == nil && o == nil
break
}
}
// Now delete the empty directories, starting from the longest path
var toDelete [ ] string
for dir , empty := range dirEmpty {
if empty {
toDelete = append ( toDelete , dir )
}
}
sort . Strings ( toDelete )
for i := len ( toDelete ) - 1 ; i >= 0 ; i -- {
dir := toDelete [ i ]
err := TryRmdir ( f , dir )
if err != nil {
Stats . Error ( )
ErrorLog ( dir , "Failed to rmdir: %v" , err )
return err
}
}
return nil
}
2016-10-23 16:34:17 +00:00
// moveOrCopyFile moves or copies a single file possibly to a new name
func moveOrCopyFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string , cp bool ) ( err error ) {
// Choose operations
Op := Move
if cp {
Op = Copy
}
// Find src object
srcObj , err := fsrc . NewObject ( srcFileName )
if err != nil {
return err
}
// Find dst object if it exists
dstObj , err := fdst . NewObject ( dstFileName )
if err == ErrorObjectNotFound {
dstObj = nil
} else if err != nil {
return err
}
if NeedTransfer ( dstObj , srcObj ) {
return Op ( fdst , dstObj , dstFileName , srcObj )
} else if ! cp {
return DeleteFile ( srcObj )
}
return nil
}
// MoveFile moves a single file possibly to a new name
func MoveFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string ) ( err error ) {
return moveOrCopyFile ( fdst , fsrc , dstFileName , srcFileName , false )
}
// CopyFile moves a single file possibly to a new name
func CopyFile ( fdst Fs , fsrc Fs , dstFileName string , srcFileName string ) ( err error ) {
return moveOrCopyFile ( fdst , fsrc , dstFileName , srcFileName , true )
}