webdav: add "fastmail" provider for Fastmail Files
This provider: - supports the `X-OC-Mtime` header to set the mtime - calculates SHA1 checksum server side and returns it as a `ME:sha1hex` prop To differentiate the new hasMESHA1 quirk, the existing hasMD5 and hasSHA1 quirks for Owncloud have been renamed to hasOCMD5 and hasOCSHA1. Fixes #6837
This commit is contained in:
Arnavion
Nick Craig-Wood
parent
0e134364ac
commit
29fe0177bd
6 changed files with 74 additions and 45 deletions
|
|
@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
|||
* Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
* Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
* Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
|
||||
* Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
|
||||
* FTP [:page_facing_up:](https://rclone.org/ftp/)
|
||||
* Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
|
||||
* Google Drive [:page_facing_up:](https://rclone.org/drive/)
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ type Prop struct {
|
|||
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
|
||||
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
|
||||
Checksums []string `xml:"prop>checksums>checksum,omitempty"`
|
||||
MESha1Hex *string `xml:"ME: prop>sha1hex,omitempty"` // Fastmail-specific sha1 checksum
|
||||
}
|
||||
|
||||
// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
|
||||
|
|
@ -102,22 +103,27 @@ func (p *Prop) StatusOK() bool {
|
|||
|
||||
// Hashes returns a map of all checksums - may be nil
|
||||
func (p *Prop) Hashes() (hashes map[hash.Type]string) {
|
||||
if len(p.Checksums) == 0 {
|
||||
return nil
|
||||
}
|
||||
hashes = make(map[hash.Type]string)
|
||||
for _, checksums := range p.Checksums {
|
||||
checksums = strings.ToLower(checksums)
|
||||
for _, checksum := range strings.Split(checksums, " ") {
|
||||
switch {
|
||||
case strings.HasPrefix(checksum, "sha1:"):
|
||||
hashes[hash.SHA1] = checksum[5:]
|
||||
case strings.HasPrefix(checksum, "md5:"):
|
||||
hashes[hash.MD5] = checksum[4:]
|
||||
if len(p.Checksums) > 0 {
|
||||
hashes = make(map[hash.Type]string)
|
||||
for _, checksums := range p.Checksums {
|
||||
checksums = strings.ToLower(checksums)
|
||||
for _, checksum := range strings.Split(checksums, " ") {
|
||||
switch {
|
||||
case strings.HasPrefix(checksum, "sha1:"):
|
||||
hashes[hash.SHA1] = checksum[5:]
|
||||
case strings.HasPrefix(checksum, "md5:"):
|
||||
hashes[hash.MD5] = checksum[4:]
|
||||
}
|
||||
}
|
||||
}
|
||||
return hashes
|
||||
} else if p.MESha1Hex != nil {
|
||||
hashes = make(map[hash.Type]string)
|
||||
hashes[hash.SHA1] = *p.MESha1Hex
|
||||
return hashes
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
return hashes
|
||||
}
|
||||
|
||||
// PropValue is a tagged name and value
|
||||
|
|
|
|||
|
|
@ -76,6 +76,9 @@ func init() {
|
|||
Name: "vendor",
|
||||
Help: "Name of the WebDAV site/service/software you are using.",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "fastmail",
|
||||
Help: "Fastmail Files",
|
||||
}, {
|
||||
Value: "nextcloud",
|
||||
Help: "Nextcloud",
|
||||
}, {
|
||||
|
|
@ -155,8 +158,9 @@ type Fs struct {
|
|||
useOCMtime bool // set if can use X-OC-Mtime
|
||||
retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default)
|
||||
checkBeforePurge bool // enables extra check that directory to purge really exists
|
||||
hasMD5 bool // set if can use owncloud style checksums for MD5
|
||||
hasSHA1 bool // set if can use owncloud style checksums for SHA1
|
||||
hasOCMD5 bool // set if can use owncloud style checksums for MD5
|
||||
hasOCSHA1 bool // set if can use owncloud style checksums for SHA1
|
||||
hasMESHA1 bool // set if can use fastmail style checksums for SHA1
|
||||
ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips
|
||||
}
|
||||
|
||||
|
|
@ -278,7 +282,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
|
|||
},
|
||||
NoRedirect: true,
|
||||
}
|
||||
if f.hasMD5 || f.hasSHA1 {
|
||||
if f.hasOCMD5 || f.hasOCSHA1 {
|
||||
opts.Body = bytes.NewBuffer(owncloudProps)
|
||||
}
|
||||
var result api.Multistatus
|
||||
|
|
@ -546,16 +550,21 @@ func (f *Fs) fetchAndSetBearerToken() error {
|
|||
// setQuirks adjusts the Fs for the vendor passed in
|
||||
func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
|
||||
switch vendor {
|
||||
case "fastmail":
|
||||
f.canStream = true
|
||||
f.precision = time.Second
|
||||
f.useOCMtime = true
|
||||
f.hasMESHA1 = true
|
||||
case "owncloud":
|
||||
f.canStream = true
|
||||
f.precision = time.Second
|
||||
f.useOCMtime = true
|
||||
f.hasMD5 = true
|
||||
f.hasSHA1 = true
|
||||
f.hasOCMD5 = true
|
||||
f.hasOCSHA1 = true
|
||||
case "nextcloud":
|
||||
f.precision = time.Second
|
||||
f.useOCMtime = true
|
||||
f.hasSHA1 = true
|
||||
f.hasOCSHA1 = true
|
||||
case "sharepoint":
|
||||
// To mount sharepoint, two Cookies are required
|
||||
// They have to be set instead of BasicAuth
|
||||
|
|
@ -667,7 +676,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
|||
"Depth": depth,
|
||||
},
|
||||
}
|
||||
if f.hasMD5 || f.hasSHA1 {
|
||||
if f.hasOCMD5 || f.hasOCSHA1 {
|
||||
opts.Body = bytes.NewBuffer(owncloudProps)
|
||||
}
|
||||
var result api.Multistatus
|
||||
|
|
@ -1126,10 +1135,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
|||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
hashes := hash.Set(hash.None)
|
||||
if f.hasMD5 {
|
||||
if f.hasOCMD5 {
|
||||
hashes.Add(hash.MD5)
|
||||
}
|
||||
if f.hasSHA1 {
|
||||
if f.hasOCSHA1 || f.hasMESHA1 {
|
||||
hashes.Add(hash.SHA1)
|
||||
}
|
||||
return hashes
|
||||
|
|
@ -1197,10 +1206,10 @@ func (o *Object) Remote() string {
|
|||
|
||||
// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t == hash.MD5 && o.fs.hasMD5 {
|
||||
if t == hash.MD5 && o.fs.hasOCMD5 {
|
||||
return o.md5, nil
|
||||
}
|
||||
if t == hash.SHA1 && o.fs.hasSHA1 {
|
||||
if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) {
|
||||
return o.sha1, nil
|
||||
}
|
||||
return "", hash.ErrUnsupported
|
||||
|
|
@ -1222,12 +1231,12 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
|
|||
o.hasMetaData = true
|
||||
o.size = info.Size
|
||||
o.modTime = time.Time(info.Modified)
|
||||
if o.fs.hasMD5 || o.fs.hasSHA1 {
|
||||
if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
|
||||
hashes := info.Hashes()
|
||||
if o.fs.hasSHA1 {
|
||||
if o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
|
||||
o.sha1 = hashes[hash.SHA1]
|
||||
}
|
||||
if o.fs.hasMD5 {
|
||||
if o.fs.hasOCMD5 {
|
||||
o.md5 = hashes[hash.MD5]
|
||||
}
|
||||
}
|
||||
|
|
@ -1315,7 +1324,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
ContentType: fs.MimeType(ctx, src),
|
||||
Options: options,
|
||||
}
|
||||
if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 {
|
||||
if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
|
||||
opts.ExtraHeaders = map[string]string{}
|
||||
if o.fs.useOCMtime {
|
||||
opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
|
||||
|
|
@ -1323,12 +1332,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
// Set one upload checksum
|
||||
// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
|
||||
// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
|
||||
if o.fs.hasSHA1 {
|
||||
if o.fs.hasOCSHA1 {
|
||||
if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
|
||||
opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
|
||||
}
|
||||
}
|
||||
if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
|
||||
if o.fs.hasOCMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
|
||||
if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
|
||||
opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
|
||||
}
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ WebDAV or S3, that work out of the box.)
|
|||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
|
||||
{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
|
||||
{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
|
||||
{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
|
||||
{{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
|
||||
{{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}
|
||||
|
|
|
|||
|
|
@ -68,9 +68,9 @@ This is an SHA256 sum of all the 4 MiB block SHA256s.
|
|||
² SFTP supports checksums if the same login has shell access and
|
||||
`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
|
||||
|
||||
³ WebDAV supports hashes when used with Owncloud and Nextcloud only.
|
||||
³ WebDAV supports hashes when used with Fastmail Files. Owncloud and Nextcloud only.
|
||||
|
||||
⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only.
|
||||
⁴ WebDAV supports modtimes when used with Fastmail Files, Owncloud and Nextcloud only.
|
||||
|
||||
⁵ [QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash) is Microsoft's own hash.
|
||||
|
||||
|
|
|
|||
|
|
@ -43,17 +43,19 @@ Choose a number from below, or type in your own value
|
|||
url> https://example.com/remote.php/webdav/
|
||||
Name of the WebDAV site/service/software you are using
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Nextcloud
|
||||
\ "nextcloud"
|
||||
2 / Owncloud
|
||||
\ "owncloud"
|
||||
3 / Sharepoint Online, authenticated by Microsoft account.
|
||||
\ "sharepoint"
|
||||
4 / Sharepoint with NTLM authentication. Usually self-hosted or on-premises.
|
||||
\ "sharepoint-ntlm"
|
||||
5 / Other site/service or software
|
||||
\ "other"
|
||||
vendor> 1
|
||||
1 / Fastmail Files
|
||||
\ (fastmail)
|
||||
2 / Nextcloud
|
||||
\ (nextcloud)
|
||||
3 / Owncloud
|
||||
\ (owncloud)
|
||||
4 / Sharepoint Online, authenticated by Microsoft account
|
||||
\ (sharepoint)
|
||||
5 / Sharepoint with NTLM authentication, usually self-hosted or on-premises
|
||||
\ (sharepoint-ntlm)
|
||||
6 / Other site/service or software
|
||||
\ (other)
|
||||
vendor> 2
|
||||
User name
|
||||
user> user
|
||||
Password.
|
||||
|
|
@ -100,10 +102,10 @@ To copy a local directory to an WebDAV directory called backup
|
|||
### Modified time and hashes ###
|
||||
|
||||
Plain WebDAV does not support modified times. However when used with
|
||||
Owncloud or Nextcloud rclone will support modified times.
|
||||
Fastmail Files, Owncloud or Nextcloud rclone will support modified times.
|
||||
|
||||
Likewise plain WebDAV does not support hashes, however when used with
|
||||
Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes.
|
||||
Fastmail Files, Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes.
|
||||
Depending on the exact version of Owncloud or Nextcloud hashes may
|
||||
appear on all objects, or only on objects which had a hash uploaded
|
||||
with them.
|
||||
|
|
@ -242,6 +244,16 @@ Properties:
|
|||
|
||||
See below for notes on specific providers.
|
||||
|
||||
## Fastmail Files
|
||||
|
||||
Use `https://webdav.fastmail.com/` or a subdirectory as the URL,
|
||||
and your Fastmail email `username@domain.tld` as the username.
|
||||
Follow [this documentation](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords)
|
||||
to create an app password with access to `Files (WebDAV)` and use
|
||||
this as the password.
|
||||
|
||||
Fastmail supports modified times using the `X-OC-Mtime` header.
|
||||
|
||||
### Owncloud
|
||||
|
||||
Click on the settings cog in the bottom right of the page and this
|
||||
|
|
|
|||
Loading…
Reference in a new issue