fs: deglobalise the config #4685
This is done by making fs.Config private and attaching it to the context instead. The Config should be obtained with fs.GetConfig and fs.AddConfig should be used to get a new mutable config that can be changed.
This commit is contained in:
Nick Craig-Wood
parent
506342317b
commit
2e21c58e6a
93 changed files with 1128 additions and 847 deletions
|
|
@ -119,13 +119,14 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.
|
|||
// Otherwise the file is considered to be not equal including if there
|
||||
// were errors reading info.
|
||||
func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
|
||||
return equal(ctx, src, dst, defaultEqualOpt())
|
||||
return equal(ctx, src, dst, defaultEqualOpt(ctx))
|
||||
}
|
||||
|
||||
// sizeDiffers compare the size of src and dst taking into account the
|
||||
// various ways of ignoring sizes
|
||||
func sizeDiffers(src, dst fs.ObjectInfo) bool {
|
||||
if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
|
||||
func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
|
||||
return false
|
||||
}
|
||||
return src.Size() != dst.Size()
|
||||
|
|
@ -142,11 +143,12 @@ type equalOpt struct {
|
|||
}
|
||||
|
||||
// default set of options for equal()
|
||||
func defaultEqualOpt() equalOpt {
|
||||
func defaultEqualOpt(ctx context.Context) equalOpt {
|
||||
ci := fs.GetConfig(ctx)
|
||||
return equalOpt{
|
||||
sizeOnly: fs.Config.SizeOnly,
|
||||
checkSum: fs.Config.CheckSum,
|
||||
updateModTime: !fs.Config.NoUpdateModTime,
|
||||
sizeOnly: ci.SizeOnly,
|
||||
checkSum: ci.CheckSum,
|
||||
updateModTime: !ci.NoUpdateModTime,
|
||||
forceModTimeMatch: false,
|
||||
}
|
||||
}
|
||||
|
|
@ -161,7 +163,8 @@ func logModTimeUpload(dst fs.Object) {
|
|||
}
|
||||
|
||||
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool {
|
||||
if sizeDiffers(src, dst) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if sizeDiffers(ctx, src, dst) {
|
||||
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
|
||||
return false
|
||||
}
|
||||
|
|
@ -218,7 +221,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
|
|||
fs.Debugf(src, "%v differ", ht)
|
||||
return false
|
||||
}
|
||||
if ht == hash.None && !fs.Config.RefreshTimes {
|
||||
if ht == hash.None && !ci.RefreshTimes {
|
||||
// if couldn't check hash, return that they differ
|
||||
return false
|
||||
}
|
||||
|
|
@ -228,7 +231,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
|
|||
if !SkipDestructive(ctx, src, "update modification time") {
|
||||
// Size and hash the same but mtime different
|
||||
// Error if objects are treated as immutable
|
||||
if fs.Config.Immutable {
|
||||
if ci.Immutable {
|
||||
fs.Errorf(dst, "StartedAt mismatch between immutable objects")
|
||||
return false
|
||||
}
|
||||
|
|
@ -243,7 +246,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
|
|||
fs.Infof(dst, "src and dst identical but can't set mod time without deleting and re-uploading")
|
||||
// Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file
|
||||
// put in the BackupDir than deleted which is what will happen if we don't delete it.
|
||||
if fs.Config.BackupDir == "" {
|
||||
if ci.BackupDir == "" {
|
||||
err = dst.Remove(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(dst, "failed to delete before re-upload: %v", err)
|
||||
|
|
@ -337,11 +340,12 @@ var _ fs.FullObjectInfo = (*OverrideRemote)(nil)
|
|||
|
||||
// CommonHash returns a single hash.Type and a HashOption with that
|
||||
// type which is in common between the two fs.Fs.
|
||||
func CommonHash(fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
|
||||
func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
// work out which hash to use - limit to 1 hash in common
|
||||
var common hash.Set
|
||||
hashType := hash.None
|
||||
if !fs.Config.IgnoreChecksum {
|
||||
if !ci.IgnoreChecksum {
|
||||
common = fb.Hashes().Overlap(fa.Hashes())
|
||||
if common.Count() > 0 {
|
||||
hashType = common.GetOne()
|
||||
|
|
@ -357,6 +361,7 @@ func CommonHash(fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
|
|||
// It returns the destination object if possible. Note that this may
|
||||
// be nil.
|
||||
func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
tr := accounting.Stats(ctx).NewTransfer(src)
|
||||
defer func() {
|
||||
tr.Done(ctx, err)
|
||||
|
|
@ -365,25 +370,25 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||
if SkipDestructive(ctx, src, "copy") {
|
||||
return newDst, nil
|
||||
}
|
||||
maxTries := fs.Config.LowLevelRetries
|
||||
maxTries := ci.LowLevelRetries
|
||||
tries := 0
|
||||
doUpdate := dst != nil
|
||||
hashType, hashOption := CommonHash(f, src.Fs())
|
||||
hashType, hashOption := CommonHash(ctx, f, src.Fs())
|
||||
|
||||
var actionTaken string
|
||||
for {
|
||||
// Try server-side copy first - if has optional interface and
|
||||
// is same underlying remote
|
||||
actionTaken = "Copied (server-side copy)"
|
||||
if fs.Config.MaxTransfer >= 0 {
|
||||
if ci.MaxTransfer >= 0 {
|
||||
var bytesSoFar int64
|
||||
if fs.Config.CutoffMode == fs.CutoffModeCautious {
|
||||
if ci.CutoffMode == fs.CutoffModeCautious {
|
||||
bytesSoFar = accounting.Stats(ctx).GetBytesWithPending() + src.Size()
|
||||
} else {
|
||||
bytesSoFar = accounting.Stats(ctx).GetBytes()
|
||||
}
|
||||
if bytesSoFar >= int64(fs.Config.MaxTransfer) {
|
||||
if fs.Config.CutoffMode == fs.CutoffModeHard {
|
||||
if bytesSoFar >= int64(ci.MaxTransfer) {
|
||||
if ci.CutoffMode == fs.CutoffModeHard {
|
||||
return nil, accounting.ErrorMaxTransferLimitReachedFatal
|
||||
}
|
||||
return nil, accounting.ErrorMaxTransferLimitReachedGraceful
|
||||
|
|
@ -408,12 +413,12 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||
}
|
||||
// If can't server-side copy, do it manually
|
||||
if err == fs.ErrorCantCopy {
|
||||
if doMultiThreadCopy(f, src) {
|
||||
if doMultiThreadCopy(ctx, f, src) {
|
||||
// Number of streams proportional to size
|
||||
streams := src.Size() / int64(fs.Config.MultiThreadCutoff)
|
||||
streams := src.Size() / int64(ci.MultiThreadCutoff)
|
||||
// With maximum
|
||||
if streams > int64(fs.Config.MultiThreadStreams) {
|
||||
streams = int64(fs.Config.MultiThreadStreams)
|
||||
if streams > int64(ci.MultiThreadStreams) {
|
||||
streams = int64(ci.MultiThreadStreams)
|
||||
}
|
||||
if streams < 2 {
|
||||
streams = 2
|
||||
|
|
@ -427,10 +432,10 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||
} else {
|
||||
var in0 io.ReadCloser
|
||||
options := []fs.OpenOption{hashOption}
|
||||
for _, option := range fs.Config.DownloadHeaders {
|
||||
for _, option := range ci.DownloadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
in0, err = NewReOpen(ctx, src, fs.Config.LowLevelRetries, options...)
|
||||
in0, err = NewReOpen(ctx, src, ci.LowLevelRetries, options...)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed to open source object")
|
||||
} else {
|
||||
|
|
@ -452,7 +457,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||
wrappedSrc = NewOverrideRemote(src, remote)
|
||||
}
|
||||
options := []fs.OpenOption{hashOption}
|
||||
for _, option := range fs.Config.UploadHeaders {
|
||||
for _, option := range ci.UploadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
if doUpdate {
|
||||
|
|
@ -491,7 +496,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||
}
|
||||
|
||||
// Verify sizes are the same after transfer
|
||||
if sizeDiffers(src, dst) {
|
||||
if sizeDiffers(ctx, src, dst) {
|
||||
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
|
||||
fs.Errorf(dst, "%v", err)
|
||||
err = fs.CountError(err)
|
||||
|
|
@ -607,16 +612,17 @@ func CanServerSideMove(fdst fs.Fs) bool {
|
|||
|
||||
// SuffixName adds the current --suffix to the remote, obeying
|
||||
// --suffix-keep-extension if set
|
||||
func SuffixName(remote string) string {
|
||||
if fs.Config.Suffix == "" {
|
||||
func SuffixName(ctx context.Context, remote string) string {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.Suffix == "" {
|
||||
return remote
|
||||
}
|
||||
if fs.Config.SuffixKeepExtension {
|
||||
if ci.SuffixKeepExtension {
|
||||
ext := path.Ext(remote)
|
||||
base := remote[:len(remote)-len(ext)]
|
||||
return base + fs.Config.Suffix + ext
|
||||
return base + ci.Suffix + ext
|
||||
}
|
||||
return remote + fs.Config.Suffix
|
||||
return remote + ci.Suffix
|
||||
}
|
||||
|
||||
// DeleteFileWithBackupDir deletes a single file respecting --dry-run
|
||||
|
|
@ -625,12 +631,13 @@ func SuffixName(remote string) string {
|
|||
// If backupDir is set then it moves the file to there instead of
|
||||
// deleting
|
||||
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
tr := accounting.Stats(ctx).NewCheckingTransfer(dst)
|
||||
defer func() {
|
||||
tr.Done(ctx, err)
|
||||
}()
|
||||
numDeletes := accounting.Stats(ctx).Deletes(1)
|
||||
if fs.Config.MaxDelete != -1 && numDeletes > fs.Config.MaxDelete {
|
||||
if ci.MaxDelete != -1 && numDeletes > ci.MaxDelete {
|
||||
return fserrors.FatalError(errors.New("--max-delete threshold reached"))
|
||||
}
|
||||
action, actioned := "delete", "Deleted"
|
||||
|
|
@ -669,11 +676,12 @@ func DeleteFile(ctx context.Context, dst fs.Object) (err error) {
|
|||
// instead of being deleted.
|
||||
func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(fs.Config.Transfers)
|
||||
ci := fs.GetConfig(ctx)
|
||||
wg.Add(ci.Transfers)
|
||||
var errorCount int32
|
||||
var fatalErrorCount int32
|
||||
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
for i := 0; i < ci.Transfers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for dst := range toBeDeleted {
|
||||
|
|
@ -779,7 +787,8 @@ func Retry(o interface{}, maxTries int, fn func() error) (err error) {
|
|||
//
|
||||
// Lists in parallel which may get them out of order
|
||||
func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error {
|
||||
return walk.ListR(ctx, f, "", false, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||
ci := fs.GetConfig(ctx)
|
||||
return walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||
entries.ForObject(fn)
|
||||
return nil
|
||||
})
|
||||
|
|
@ -898,8 +907,9 @@ func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error)
|
|||
}
|
||||
|
||||
// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing.
|
||||
func ConfigMaxDepth(recursive bool) int {
|
||||
depth := fs.Config.MaxDepth
|
||||
func ConfigMaxDepth(ctx context.Context, recursive bool) int {
|
||||
ci := fs.GetConfig(ctx)
|
||||
depth := ci.MaxDepth
|
||||
if !recursive && depth < 0 {
|
||||
depth = 1
|
||||
}
|
||||
|
|
@ -908,7 +918,7 @@ func ConfigMaxDepth(recursive bool) int {
|
|||
|
||||
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
|
||||
func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
|
||||
return walk.ListR(ctx, f, "", false, ConfigMaxDepth(false), walk.ListDirs, func(entries fs.DirEntries) error {
|
||||
return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error {
|
||||
entries.ForDir(func(dir fs.Directory) {
|
||||
if dir != nil {
|
||||
syncFprintf(w, "%12d %13s %9d %s\n", dir.Size(), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), dir.Items(), dir.Remote())
|
||||
|
|
@ -985,7 +995,8 @@ func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {
|
|||
// Delete removes all the contents of a container. Unlike Purge, it
|
||||
// obeys includes and excludes.
|
||||
func Delete(ctx context.Context, f fs.Fs) error {
|
||||
delChan := make(fs.ObjectsChan, fs.Config.Transfers)
|
||||
ci := fs.GetConfig(ctx)
|
||||
delChan := make(fs.ObjectsChan, ci.Transfers)
|
||||
delErr := make(chan error, 1)
|
||||
go func() {
|
||||
delErr <- DeleteFiles(ctx, delChan)
|
||||
|
|
@ -1008,10 +1019,11 @@ func Delete(ctx context.Context, f fs.Fs) error {
|
|||
//
|
||||
// If the error was ErrorDirNotFound then it will be ignored
|
||||
func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {
|
||||
o := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
ci := fs.GetConfig(ctx)
|
||||
o := make(fs.ObjectsChan, ci.Checkers)
|
||||
go func() {
|
||||
defer close(o)
|
||||
err := walk.ListR(ctx, f, dir, true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||
err := walk.ListR(ctx, f, dir, true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
|
||||
entries.ForObject(func(obj fs.Object) {
|
||||
o <- obj
|
||||
})
|
||||
|
|
@ -1054,6 +1066,7 @@ type readCloser struct {
|
|||
// if count >= 0 then only that many characters will be output
|
||||
func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
|
||||
var mu sync.Mutex
|
||||
ci := fs.GetConfig(ctx)
|
||||
return ListFn(ctx, f, func(o fs.Object) {
|
||||
var err error
|
||||
tr := accounting.Stats(ctx).NewTransfer(o)
|
||||
|
|
@ -1072,7 +1085,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
|
|||
if opt.Start > 0 || opt.End >= 0 {
|
||||
options = append(options, &opt)
|
||||
}
|
||||
for _, option := range fs.Config.DownloadHeaders {
|
||||
for _, option := range ci.DownloadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
in, err := o.Open(ctx, options...)
|
||||
|
|
@ -1098,6 +1111,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
|
|||
|
||||
// Rcat reads data from the Reader until EOF and uploads it to a file on remote
|
||||
func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1)
|
||||
defer func() {
|
||||
tr.Done(ctx, err)
|
||||
|
|
@ -1108,7 +1122,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
|
|||
var trackingIn io.Reader
|
||||
var hasher *hash.MultiHasher
|
||||
var options []fs.OpenOption
|
||||
if !fs.Config.IgnoreChecksum {
|
||||
if !ci.IgnoreChecksum {
|
||||
hashes := hash.NewHashSet(fdst.Hashes().GetOne()) // just pick one hash
|
||||
hashOption := &fs.HashesOption{Hashes: hashes}
|
||||
options = append(options, hashOption)
|
||||
|
|
@ -1120,7 +1134,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
|
|||
} else {
|
||||
trackingIn = readCounter
|
||||
}
|
||||
for _, option := range fs.Config.UploadHeaders {
|
||||
for _, option := range ci.UploadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
|
||||
|
|
@ -1140,7 +1154,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
|
|||
}
|
||||
|
||||
// check if file small enough for direct upload
|
||||
buf := make([]byte, fs.Config.StreamingUploadCutoff)
|
||||
buf := make([]byte, ci.StreamingUploadCutoff)
|
||||
if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", n)
|
||||
src := object.NewMemoryObject(dstFileName, modTime, buf[:n])
|
||||
|
|
@ -1202,9 +1216,10 @@ func PublicLink(ctx context.Context, f fs.Fs, remote string, expire fs.Duration,
|
|||
// Rmdirs removes any empty directories (or directories only
|
||||
// containing empty directories) under f, including f.
|
||||
func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
|
||||
ci := fs.GetConfig(ctx)
|
||||
dirEmpty := make(map[string]bool)
|
||||
dirEmpty[dir] = !leaveRoot
|
||||
err := walk.Walk(ctx, f, dir, true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
err := walk.Walk(ctx, f, dir, true, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
if err != nil {
|
||||
err = fs.CountError(err)
|
||||
fs.Errorf(f, "Failed to list %q: %v", dirPath, err)
|
||||
|
|
@ -1263,9 +1278,10 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
|
|||
|
||||
// GetCompareDest sets up --compare-dest
|
||||
func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) {
|
||||
CompareDest, err = cache.Get(ctx, fs.Config.CompareDest)
|
||||
ci := fs.GetConfig(ctx)
|
||||
CompareDest, err = cache.Get(ctx, ci.CompareDest)
|
||||
if err != nil {
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err))
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err))
|
||||
}
|
||||
return CompareDest, nil
|
||||
}
|
||||
|
|
@ -1299,9 +1315,10 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No
|
|||
|
||||
// GetCopyDest sets up --copy-dest
|
||||
func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) {
|
||||
CopyDest, err = cache.Get(ctx, fs.Config.CopyDest)
|
||||
ci := fs.GetConfig(ctx)
|
||||
CopyDest, err = cache.Get(ctx, ci.CopyDest)
|
||||
if err != nil {
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err))
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err))
|
||||
}
|
||||
if !SameConfig(fdst, CopyDest) {
|
||||
return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
|
||||
|
|
@ -1332,7 +1349,7 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
|
|||
default:
|
||||
return false, err
|
||||
}
|
||||
opt := defaultEqualOpt()
|
||||
opt := defaultEqualOpt(ctx)
|
||||
opt.updateModTime = false
|
||||
if equal(ctx, src, CopyDestFile, opt) {
|
||||
if dst == nil || !Equal(ctx, src, dst) {
|
||||
|
|
@ -1364,9 +1381,10 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
|
|||
//
|
||||
// Returns True if src does not need to be copied
|
||||
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
|
||||
if fs.Config.CompareDest != "" {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.CompareDest != "" {
|
||||
return compareDest(ctx, dst, src, CompareOrCopyDest)
|
||||
} else if fs.Config.CopyDest != "" {
|
||||
} else if ci.CopyDest != "" {
|
||||
return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
|
||||
}
|
||||
return false, nil
|
||||
|
|
@ -1378,22 +1396,23 @@ func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, Comp
|
|||
// Returns a flag which indicates whether the file needs to be
|
||||
// transferred or not.
|
||||
func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if dst == nil {
|
||||
fs.Debugf(src, "Need to transfer - File not found at Destination")
|
||||
return true
|
||||
}
|
||||
// If we should ignore existing files, don't transfer
|
||||
if fs.Config.IgnoreExisting {
|
||||
if ci.IgnoreExisting {
|
||||
fs.Debugf(src, "Destination exists, skipping")
|
||||
return false
|
||||
}
|
||||
// If we should upload unconditionally
|
||||
if fs.Config.IgnoreTimes {
|
||||
if ci.IgnoreTimes {
|
||||
fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use")
|
||||
return true
|
||||
}
|
||||
// If UpdateOlder is in effect, skip if dst is newer than src
|
||||
if fs.Config.UpdateOlder {
|
||||
if ci.UpdateOlder {
|
||||
srcModTime := src.ModTime(ctx)
|
||||
dstModTime := dst.ModTime(ctx)
|
||||
dt := dstModTime.Sub(srcModTime)
|
||||
|
|
@ -1411,7 +1430,7 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
|
|||
return false
|
||||
case dt <= -modifyWindow:
|
||||
// force --checksum on for the check and do update modtimes by default
|
||||
opt := defaultEqualOpt()
|
||||
opt := defaultEqualOpt(ctx)
|
||||
opt.forceModTimeMatch = true
|
||||
if equal(ctx, src, dst, opt) {
|
||||
fs.Debugf(src, "Unchanged skipping")
|
||||
|
|
@ -1419,8 +1438,8 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
|
|||
}
|
||||
default:
|
||||
// Do a size only compare unless --checksum is set
|
||||
opt := defaultEqualOpt()
|
||||
opt.sizeOnly = !fs.Config.CheckSum
|
||||
opt := defaultEqualOpt(ctx)
|
||||
opt.sizeOnly = !ci.CheckSum
|
||||
if equal(ctx, src, dst, opt) {
|
||||
fs.Debugf(src, "Destination mod time is within %v of source and files identical, skipping", modifyWindow)
|
||||
return false
|
||||
|
|
@ -1483,7 +1502,7 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,
|
|||
|
||||
// copyURLFn copies the data from the url to the function supplied
|
||||
func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) {
|
||||
client := fshttp.NewClient(fs.Config)
|
||||
client := fshttp.NewClient(fs.GetConfig(ctx))
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -1531,10 +1550,11 @@ func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error)
|
|||
|
||||
// BackupDir returns the correctly configured --backup-dir
|
||||
func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) {
|
||||
if fs.Config.BackupDir != "" {
|
||||
backupDir, err = cache.Get(ctx, fs.Config.BackupDir)
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.BackupDir != "" {
|
||||
backupDir, err = cache.Get(ctx, ci.BackupDir)
|
||||
if err != nil {
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", fs.Config.BackupDir, err))
|
||||
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", ci.BackupDir, err))
|
||||
}
|
||||
if !SameConfig(fdst, backupDir) {
|
||||
return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination"))
|
||||
|
|
@ -1547,7 +1567,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
|
|||
return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap"))
|
||||
}
|
||||
} else {
|
||||
if fs.Config.Suffix == "" {
|
||||
if ci.Suffix == "" {
|
||||
if SameDir(fdst, backupDir) {
|
||||
return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't be the same"))
|
||||
}
|
||||
|
|
@ -1556,7 +1576,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
|
|||
}
|
||||
}
|
||||
}
|
||||
} else if fs.Config.Suffix != "" {
|
||||
} else if ci.Suffix != "" {
|
||||
// --backup-dir is not set but --suffix is - use the destination as the backupDir
|
||||
backupDir = fdst
|
||||
} else {
|
||||
|
|
@ -1570,7 +1590,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
|
|||
|
||||
// MoveBackupDir moves a file to the backup dir
|
||||
func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error) {
|
||||
remoteWithSuffix := SuffixName(dst.Remote())
|
||||
remoteWithSuffix := SuffixName(ctx, dst.Remote())
|
||||
overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix)
|
||||
_, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst)
|
||||
return err
|
||||
|
|
@ -1578,6 +1598,7 @@ func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err err
|
|||
|
||||
// moveOrCopyFile moves or copies a single file possibly to a new name
|
||||
func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
dstFilePath := path.Join(fdst.Root(), dstFileName)
|
||||
srcFilePath := path.Join(fsrc.Root(), srcFileName)
|
||||
if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {
|
||||
|
|
@ -1599,7 +1620,7 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
|
|||
|
||||
// Find dst object if it exists
|
||||
var dstObj fs.Object
|
||||
if !fs.Config.NoCheckDest {
|
||||
if !ci.NoCheckDest {
|
||||
dstObj, err = fdst.NewObject(ctx, dstFileName)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
dstObj = nil
|
||||
|
|
@ -1635,18 +1656,18 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
|
|||
}
|
||||
|
||||
var backupDir, copyDestDir fs.Fs
|
||||
if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
|
||||
if ci.BackupDir != "" || ci.Suffix != "" {
|
||||
backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating Fs for --backup-dir failed")
|
||||
}
|
||||
}
|
||||
if fs.Config.CompareDest != "" {
|
||||
if ci.CompareDest != "" {
|
||||
copyDestDir, err = GetCompareDest(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if fs.Config.CopyDest != "" {
|
||||
} else if ci.CopyDest != "" {
|
||||
copyDestDir, err = GetCopyDest(ctx, fdst)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -1853,6 +1874,7 @@ func (l *ListFormat) Format(entry *ListJSONItem) (result string) {
|
|||
// It does this by loading the directory tree into memory (using ListR
|
||||
// if available) and doing renames in parallel.
|
||||
func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
// Use DirMove if possible
|
||||
if doDirMove := f.Features().DirMove; doDirMove != nil {
|
||||
err = doDirMove(ctx, f, srcRemote, dstRemote)
|
||||
|
|
@ -1885,9 +1907,9 @@ func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err err
|
|||
o fs.Object
|
||||
newPath string
|
||||
}
|
||||
renames := make(chan rename, fs.Config.Transfers)
|
||||
renames := make(chan rename, ci.Transfers)
|
||||
g, gCtx := errgroup.WithContext(context.Background())
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
for i := 0; i < ci.Transfers; i++ {
|
||||
g.Go(func() error {
|
||||
for job := range renames {
|
||||
dstOverwritten, _ := f.NewObject(gCtx, job.newPath)
|
||||
|
|
@ -2019,11 +2041,12 @@ func skipDestructiveChoose(ctx context.Context, subject interface{}, action stri
|
|||
// to action subject".
|
||||
func SkipDestructive(ctx context.Context, subject interface{}, action string) (skip bool) {
|
||||
var flag string
|
||||
ci := fs.GetConfig(ctx)
|
||||
switch {
|
||||
case fs.Config.DryRun:
|
||||
case ci.DryRun:
|
||||
flag = "--dry-run"
|
||||
skip = true
|
||||
case fs.Config.Interactive:
|
||||
case ci.Interactive:
|
||||
flag = "--interactive"
|
||||
interactiveMu.Lock()
|
||||
defer interactiveMu.Unlock()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue