operations: ignore size of objects when they are < 0 #320
This allows google docs to be transferred and checked correctly.
This commit is contained in:
parent
44276db454
commit
8df78f2b6d
3 changed files with 67 additions and 30 deletions
|
@ -382,13 +382,20 @@ see User rate limit exceeded errors, wait at least 24 hours and retry.
|
||||||
You can disable server side copies with `--disable copy` to download
|
You can disable server side copies with `--disable copy` to download
|
||||||
and upload the files if you prefer.
|
and upload the files if you prefer.
|
||||||
|
|
||||||
|
#### Limitations of Google Docs ####
|
||||||
|
|
||||||
Google docs will appear as size -1 in `rclone ls` and as size 0 in
|
Google docs will appear as size -1 in `rclone ls` and as size 0 in
|
||||||
anything which uses the VFS layer, eg `rclone mount`, `rclone serve
|
anything which uses the VFS layer, eg `rclone mount`, `rclone serve`.
|
||||||
XXX`. This is because rclone can't find out the size of the Google
|
|
||||||
Documents until they are downloaded. An unfortunate consequence of
|
This is because rclone can't find out the size of the Google docs
|
||||||
this is that you can't download Google docs using `rclone mount` - you
|
without downloading them.
|
||||||
will get a 0 sized file. If you try again the doc may gain its
|
|
||||||
correct size and be downloadable.
|
Google docs will transfer correctly with `rclone sync`, `rclone copy`
|
||||||
|
etc as rclone knows to ignore the size when doing the transfer.
|
||||||
|
|
||||||
|
However an unfortunate consequence of this is that you can't download
|
||||||
|
Google docs using `rclone mount` - you will get a 0 sized file. If
|
||||||
|
you try again the doc may gain its correct size and be downloadable.
|
||||||
|
|
||||||
### Duplicated files ###
|
### Duplicated files ###
|
||||||
|
|
||||||
|
@ -406,23 +413,9 @@ Android duplicates files on drive sometimes.
|
||||||
|
|
||||||
### Rclone appears to be re-copying files it shouldn't ###
|
### Rclone appears to be re-copying files it shouldn't ###
|
||||||
|
|
||||||
There are two possible reasons for rclone to recopy files which
|
The most likely cause of this is the duplicated file issue above - run
|
||||||
haven't changed to Google Drive.
|
`rclone dedupe` and check your logs for duplicate object or directory
|
||||||
|
messages.
|
||||||
The first is the duplicated file issue above - run `rclone dedupe` and
|
|
||||||
check your logs for duplicate object or directory messages.
|
|
||||||
|
|
||||||
The second is that sometimes Google reports different sizes for the
|
|
||||||
Google Docs exports which will cause rclone to re-download Google Docs
|
|
||||||
for no apparent reason. `--ignore-size` is a not very satisfactory
|
|
||||||
work-around for this if it is causing you a lot of problems.
|
|
||||||
|
|
||||||
### Google docs downloads sometimes fail with "Failed to copy: read X bytes expecting Y" ###
|
|
||||||
|
|
||||||
This is the same problem as above. Google reports the google doc is
|
|
||||||
one size, but rclone downloads a different size. Work-around with the
|
|
||||||
`--ignore-size` flag or wait for rclone to retry the download which it
|
|
||||||
will.
|
|
||||||
|
|
||||||
### Making your own client_id ###
|
### Making your own client_id ###
|
||||||
|
|
||||||
|
|
|
@ -96,12 +96,19 @@ func Equal(src fs.ObjectInfo, dst fs.Object) bool {
|
||||||
return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
|
return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sizeDiffers compare the size of src and dst taking into account the
|
||||||
|
// various ways of ignoring sizes
|
||||||
|
func sizeDiffers(src, dst fs.ObjectInfo) bool {
|
||||||
|
if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return src.Size() != dst.Size()
|
||||||
|
}
|
||||||
|
|
||||||
func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
|
func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
|
||||||
if !fs.Config.IgnoreSize {
|
if sizeDiffers(src, dst) {
|
||||||
if src.Size() != dst.Size() {
|
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
|
||||||
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
|
return false
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if sizeOnly {
|
if sizeOnly {
|
||||||
fs.Debugf(src, "Sizes identical")
|
fs.Debugf(src, "Sizes identical")
|
||||||
|
@ -310,7 +317,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify sizes are the same after transfer
|
// Verify sizes are the same after transfer
|
||||||
if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
|
if sizeDiffers(src, dst) {
|
||||||
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
|
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
|
||||||
fs.Errorf(dst, "%v", err)
|
fs.Errorf(dst, "%v", err)
|
||||||
fs.CountError(err)
|
fs.CountError(err)
|
||||||
|
@ -599,7 +606,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||||
func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
|
func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
|
||||||
accounting.Stats.Checking(src.Remote())
|
accounting.Stats.Checking(src.Remote())
|
||||||
defer accounting.Stats.DoneChecking(src.Remote())
|
defer accounting.Stats.DoneChecking(src.Remote())
|
||||||
if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
|
if sizeDiffers(src, dst) {
|
||||||
err := errors.Errorf("Sizes differ")
|
err := errors.Errorf("Sizes differ")
|
||||||
fs.Errorf(src, "%v", err)
|
fs.Errorf(src, "%v", err)
|
||||||
fs.CountError(err)
|
fs.CountError(err)
|
||||||
|
|
|
@ -1,3 +1,40 @@
|
||||||
// Internal tests for operations
|
// Internal tests for operations
|
||||||
|
|
||||||
package operations
|
package operations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/ncw/rclone/fs"
|
||||||
|
"github.com/ncw/rclone/fs/object"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSizeDiffers(t *testing.T) {
|
||||||
|
when := time.Now()
|
||||||
|
for _, test := range []struct {
|
||||||
|
ignoreSize bool
|
||||||
|
srcSize int64
|
||||||
|
dstSize int64
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{false, 0, 0, false},
|
||||||
|
{false, 1, 2, true},
|
||||||
|
{false, 1, -1, false},
|
||||||
|
{false, -1, 1, false},
|
||||||
|
{true, 0, 0, false},
|
||||||
|
{true, 1, 2, false},
|
||||||
|
{true, 1, -1, false},
|
||||||
|
{true, -1, 1, false},
|
||||||
|
} {
|
||||||
|
src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil)
|
||||||
|
dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil)
|
||||||
|
oldIgnoreSize := fs.Config.IgnoreSize
|
||||||
|
fs.Config.IgnoreSize = test.ignoreSize
|
||||||
|
got := sizeDiffers(src, dst)
|
||||||
|
fs.Config.IgnoreSize = oldIgnoreSize
|
||||||
|
assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue