drive: add support of SHA-1 and SHA-256 checksum
This commit is contained in:
parent
9e80d48b03
commit
8fd66daab6
3 changed files with 72 additions and 56 deletions
|
@ -71,7 +71,7 @@ const (
|
||||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||||
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
||||||
defaultChunkSize = 8 * fs.Mebi
|
defaultChunkSize = 8 * fs.Mebi
|
||||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
partialFields = "id,name,size,md5Checksum,sha1Checksum,sha256Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
||||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||||
defaultXDGIcon = "text-html"
|
defaultXDGIcon = "text-html"
|
||||||
|
@ -323,13 +323,13 @@ rather than shortcuts themselves when doing server side copies.`,
|
||||||
}, {
|
}, {
|
||||||
Name: "skip_checksum_gphotos",
|
Name: "skip_checksum_gphotos",
|
||||||
Default: false,
|
Default: false,
|
||||||
Help: `Skip MD5 checksum on Google photos and videos only.
|
Help: `Skip checksums on Google photos and videos only.
|
||||||
|
|
||||||
Use this if you get checksum errors when transferring Google photos or
|
Use this if you get checksum errors when transferring Google photos or
|
||||||
videos.
|
videos.
|
||||||
|
|
||||||
Setting this flag will cause Google photos and videos to return a
|
Setting this flag will cause Google photos and videos to return a
|
||||||
blank MD5 checksum.
|
blank checksums.
|
||||||
|
|
||||||
Google photos are identified by being in the "photos" space.
|
Google photos are identified by being in the "photos" space.
|
||||||
|
|
||||||
|
@ -751,6 +751,8 @@ type Object struct {
|
||||||
baseObject
|
baseObject
|
||||||
url string // Download URL of this object
|
url string // Download URL of this object
|
||||||
md5sum string // md5sum of the object
|
md5sum string // md5sum of the object
|
||||||
|
sha1sum string // sha1sum of the object
|
||||||
|
sha256sum string // sha256sum of the object
|
||||||
v2Download bool // generate v2 download link ondemand
|
v2Download bool // generate v2 download link ondemand
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1414,6 +1416,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||||
for _, space := range info.Spaces {
|
for _, space := range info.Spaces {
|
||||||
if space == "photos" {
|
if space == "photos" {
|
||||||
info.Md5Checksum = ""
|
info.Md5Checksum = ""
|
||||||
|
info.Sha1Checksum = ""
|
||||||
|
info.Sha256Checksum = ""
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1422,6 +1426,8 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||||
baseObject: f.newBaseObject(remote, info),
|
baseObject: f.newBaseObject(remote, info),
|
||||||
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
||||||
md5sum: strings.ToLower(info.Md5Checksum),
|
md5sum: strings.ToLower(info.Md5Checksum),
|
||||||
|
sha1sum: strings.ToLower(info.Sha1Checksum),
|
||||||
|
sha256sum: strings.ToLower(info.Sha256Checksum),
|
||||||
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
||||||
}
|
}
|
||||||
if info.ResourceKey != "" {
|
if info.ResourceKey != "" {
|
||||||
|
@ -3007,7 +3013,7 @@ func (f *Fs) DirCacheFlush() {
|
||||||
|
|
||||||
// Hashes returns the supported hash sets.
|
// Hashes returns the supported hash sets.
|
||||||
func (f *Fs) Hashes() hash.Set {
|
func (f *Fs) Hashes() hash.Set {
|
||||||
return hash.Set(hash.MD5)
|
return hash.NewHashSet(hash.MD5, hash.SHA1, hash.SHA256)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
|
func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
|
||||||
|
@ -3568,11 +3574,17 @@ func (o *baseObject) Remote() string {
|
||||||
|
|
||||||
// Hash returns the Md5sum of an object returning a lowercase hex string
|
// Hash returns the Md5sum of an object returning a lowercase hex string
|
||||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t != hash.MD5 {
|
if t == hash.MD5 {
|
||||||
return "", hash.ErrUnsupported
|
|
||||||
}
|
|
||||||
return o.md5sum, nil
|
return o.md5sum, nil
|
||||||
}
|
}
|
||||||
|
if t == hash.SHA1 {
|
||||||
|
return o.sha1sum, nil
|
||||||
|
}
|
||||||
|
if t == hash.SHA256 {
|
||||||
|
return o.sha256sum, nil
|
||||||
|
}
|
||||||
|
return "", hash.ErrUnsupported
|
||||||
|
}
|
||||||
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
|
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||||
if t != hash.MD5 {
|
if t != hash.MD5 {
|
||||||
return "", hash.ErrUnsupported
|
return "", hash.ErrUnsupported
|
||||||
|
|
|
@ -772,13 +772,13 @@ Properties:
|
||||||
|
|
||||||
#### --drive-skip-checksum-gphotos
|
#### --drive-skip-checksum-gphotos
|
||||||
|
|
||||||
Skip MD5 checksum on Google photos and videos only.
|
Skip checksums on Google photos and videos only.
|
||||||
|
|
||||||
Use this if you get checksum errors when transferring Google photos or
|
Use this if you get checksum errors when transferring Google photos or
|
||||||
videos.
|
videos.
|
||||||
|
|
||||||
Setting this flag will cause Google photos and videos to return a
|
Setting this flag will cause Google photos and videos to return a
|
||||||
blank MD5 checksum.
|
blank checksums.
|
||||||
|
|
||||||
Google photos are identified by being in the "photos" space.
|
Google photos are identified by being in the "photos" space.
|
||||||
|
|
||||||
|
@ -1526,6 +1526,10 @@ Waiting a moderate period of time between attempts (estimated to be
|
||||||
approximately 1 hour) and/or not using --fast-list both seem to be
|
approximately 1 hour) and/or not using --fast-list both seem to be
|
||||||
effective in preventing the problem.
|
effective in preventing the problem.
|
||||||
|
|
||||||
|
### Hashes
|
||||||
|
|
||||||
|
We need to say that all files have MD5 hashes, but a small fraction of files uploaded may not have SHA1 or SHA256 hashes especially if they were uploaded before 2018.
|
||||||
|
|
||||||
## Making your own client_id
|
## Making your own client_id
|
||||||
|
|
||||||
When you use rclone with Google drive in its default configuration you
|
When you use rclone with Google drive in its default configuration you
|
||||||
|
|
|
@ -15,7 +15,7 @@ show through.
|
||||||
Here is an overview of the major features of each cloud storage system.
|
Here is an overview of the major features of each cloud storage system.
|
||||||
|
|
||||||
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
|
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata |
|
||||||
| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
|
| ---------------------------- |:-----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
|
||||||
| 1Fichier | Whirlpool | - | No | Yes | R | - |
|
| 1Fichier | Whirlpool | - | No | Yes | R | - |
|
||||||
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
|
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
|
||||||
| Amazon Drive | MD5 | - | Yes | No | R | - |
|
| Amazon Drive | MD5 | - | Yes | No | R | - |
|
||||||
|
@ -27,7 +27,7 @@ Here is an overview of the major features of each cloud storage system.
|
||||||
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
|
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
|
||||||
| FTP | - | R/W ¹⁰ | No | No | - | - |
|
| FTP | - | R/W ¹⁰ | No | No | - | - |
|
||||||
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
|
| Google Cloud Storage | MD5 | R/W | No | No | R/W | - |
|
||||||
| Google Drive | MD5 | R/W | No | Yes | R/W | - |
|
| Google Drive | MD5, SHA1, SHA256 | R/W | No | Yes | R/W | - |
|
||||||
| Google Photos | - | - | No | Yes | R | - |
|
| Google Photos | - | - | No | Yes | R | - |
|
||||||
| HDFS | - | R/W | No | No | - | - |
|
| HDFS | - | R/W | No | No | - | - |
|
||||||
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
|
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
|
||||||
|
|
Loading…
Reference in a new issue