crypt: calculate hashes for uploads from local disk

Before this change crypt would not calculate hashes for files it was
uploading. This is because, in the general case, they have to be
downloaded, encrypted and hashed which is too resource intensive.

However this causes backends which need the hash first before
uploading (eg s3/b2 when uploading chunked files) not to have a hash
of the file. This causes cryptcheck to complain about missing hashes
on large files uploaded via s3/b2.

This change calculates hashes for the upload if the upload is coming
from a local filesystem. It does this by encrypting and hashing the
local file re-using the code used by cryptcheck. For a local disk this
is not a lot more intensive than calculating the hash.

See: https://forum.rclone.org/t/strange-output-for-cryptcheck/15437
Fixes: #2809
This commit is contained in:
Nick Craig-Wood 2020-04-09 16:43:34 +01:00
parent 44b1a591a8
commit 1648c1a0f3
3 changed files with 210 additions and 33 deletions

View file

@ -510,6 +510,7 @@ func (c *Cipher) DecryptDirName(in string) (string, error) {
return c.decryptFileName(in)
}
// NameEncryptionMode returns the encryption mode in use for names
func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
return c.mode
}
@ -660,13 +661,19 @@ func (fh *encrypter) finish(err error) (int, error) {
}
// Encrypt data encrypts the data stream
func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
out, err := c.newEncrypter(in, nil)
if err != nil {
return nil, err
return nil, nil, err
}
return wrap(out), nil // and wrap the accounting back on
return wrap(out), out, nil // and wrap the accounting back on
}
// EncryptData encrypts the data stream
func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
out, _, err := c.encryptData(in)
return out, err
}
// decrypter decrypts an io.ReaderCloser on the fly