operations: ignore size of objects when they are < 0 #320

This allows google docs to be transferred and checked correctly.
This commit is contained in:
Nick Craig-Wood 2018-01-31 16:15:30 +00:00
parent 44276db454
commit 8df78f2b6d
3 changed files with 67 additions and 30 deletions

View file

@ -382,13 +382,20 @@ see User rate limit exceeded errors, wait at least 24 hours and retry.
You can disable server side copies with `--disable copy` to download
and upload the files if you prefer.
#### Limitations of Google Docs ####
Google docs will appear as size -1 in `rclone ls` and as size 0 in
anything which uses the VFS layer, eg `rclone mount`, `rclone serve
XXX`. This is because rclone can't find out the size of the Google
Documents until they are downloaded. An unfortunate consequence of
this is that you can't download Google docs using `rclone mount` - you
will get a 0 sized file. If you try again the doc may gain its
correct size and be downloadable.
anything which uses the VFS layer, eg `rclone mount`, `rclone serve`.
This is because rclone can't find out the size of the Google docs
without downloading them.
Google docs will transfer correctly with `rclone sync`, `rclone copy`
etc as rclone knows to ignore the size when doing the transfer.
However an unfortunate consequence of this is that you can't download
Google docs using `rclone mount` - you will get a 0 sized file. If
you try again the doc may gain its correct size and be downloadable.
### Duplicated files ###
@ -406,23 +413,9 @@ Android duplicates files on drive sometimes.
### Rclone appears to be re-copying files it shouldn't ###
There are two possible reasons for rclone to recopy files which
haven't changed to Google Drive.
The first is the duplicated file issue above - run `rclone dedupe` and
check your logs for duplicate object or directory messages.
The second is that sometimes Google reports different sizes for the
Google Docs exports which will cause rclone to re-download Google Docs
for no apparent reason. `--ignore-size` is a not very satisfactory
work-around for this if it is causing you a lot of problems.
### Google docs downloads sometimes fail with "Failed to copy: read X bytes expecting Y" ###
This is the same problem as above. Google reports the google doc is
one size, but rclone downloads a different size. Work-around with the
`--ignore-size` flag or wait for rclone to retry the download which it
will.
The most likely cause of this is the duplicated file issue above - run
`rclone dedupe` and check your logs for duplicate object or directory
messages.
### Making your own client_id ###

View file

@ -96,12 +96,19 @@ func Equal(src fs.ObjectInfo, dst fs.Object) bool {
return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
}
func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
if !fs.Config.IgnoreSize {
if src.Size() != dst.Size() {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
// sizeDiffers compare the size of src and dst taking into account the
// various ways of ignoring sizes
func sizeDiffers(src, dst fs.ObjectInfo) bool {
if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
return false
}
return src.Size() != dst.Size()
}
func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
if sizeDiffers(src, dst) {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
return false
}
if sizeOnly {
fs.Debugf(src, "Sizes identical")
@ -310,7 +317,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
}
// Verify sizes are the same after transfer
if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
if sizeDiffers(src, dst) {
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
fs.Errorf(dst, "%v", err)
fs.CountError(err)
@ -599,7 +606,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
accounting.Stats.Checking(src.Remote())
defer accounting.Stats.DoneChecking(src.Remote())
if !fs.Config.IgnoreSize && src.Size() != dst.Size() {
if sizeDiffers(src, dst) {
err := errors.Errorf("Sizes differ")
fs.Errorf(src, "%v", err)
fs.CountError(err)

View file

@ -1,3 +1,40 @@
// Internal tests for operations
package operations
import (
"fmt"
"testing"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/object"
"github.com/stretchr/testify/assert"
)
func TestSizeDiffers(t *testing.T) {
when := time.Now()
for _, test := range []struct {
ignoreSize bool
srcSize int64
dstSize int64
want bool
}{
{false, 0, 0, false},
{false, 1, 2, true},
{false, 1, -1, false},
{false, -1, 1, false},
{true, 0, 0, false},
{true, 1, 2, false},
{true, 1, -1, false},
{true, -1, 1, false},
} {
src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil)
dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil)
oldIgnoreSize := fs.Config.IgnoreSize
fs.Config.IgnoreSize = test.ignoreSize
got := sizeDiffers(src, dst)
fs.Config.IgnoreSize = oldIgnoreSize
assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
}
}