Add cryptcheck command to check integrity of crypt remotes #1102

cmd/all/all.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/config"
 	_ "github.com/ncw/rclone/cmd/copy"
 	_ "github.com/ncw/rclone/cmd/copyto"
+	_ "github.com/ncw/rclone/cmd/cryptcheck"
 	_ "github.com/ncw/rclone/cmd/dedupe"
 	_ "github.com/ncw/rclone/cmd/delete"
 	_ "github.com/ncw/rclone/cmd/genautocomplete"

cmd/cryptcheck/cryptcheck.go

@@ -0,0 +1,108 @@
+package cryptcheck
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/crypt"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "cryptcheck remote:path cryptedremote:path",
+	Short: `Cryptcheck checks the integritity of a crypted remote.`,
+	Long: `
+rclone cryptcheck checks a remote against a crypted remote.  This is
+the equivalent of running rclone check, but able to check the
+checksums of the crypted remote.
+
+For it to work the underlying remote of the cryptedremote must support
+some kind of checksum.
+
+It works by reading the nonce from each file on the cryptedremote: and
+using that to encrypt each file on the remote:.  It then checks the
+checksum of the underlying file on the cryptedremote: against the
+checksum of the file it has just encrypted.
+
+Use it like this
+
+    rclone cryptcheck /path/to/files encryptedremote:path
+
+You can use it like this also, but that will involve downloading all
+the files in remote:path.
+
+    rclone cryptcheck remote:path encryptedremote:path
+
+After it has run it will log the status of the encryptedremote:.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(2, 2, command, args)
+		fsrc, fdst := cmd.NewFsSrcDst(args)
+		cmd.Run(false, true, command, func() error {
+			return cryptCheck(fdst, fsrc)
+		})
+	},
+}
+
+// cryptCheck checks the integrity of a crypted remote
+func cryptCheck(fdst, fsrc fs.Fs) error {
+	// Check to see fcrypt is a crypt
+	fcrypt, ok := fdst.(*crypt.Fs)
+	if !ok {
+		return errors.Errorf("%s:%s is not a crypt remote", fdst.Name(), fdst.Root())
+	}
+	// Find a hash to use
+	funderlying := fcrypt.UnWrap()
+	hashType := funderlying.Hashes().GetOne()
+	if hashType == fs.HashNone {
+		return errors.Errorf("%s:%s does not support any hashes", funderlying.Name(), funderlying.Root())
+	}
+	fs.Infof(nil, "Using %v for hash comparisons", hashType)
+
+	// checkIdentical checks to see if dst and src are identical
+	//
+	// it returns true if differences were found
+	// it also returns whether it couldn't be hashed
+	checkIdentical := func(dst, src fs.Object) (differ bool, noHash bool) {
+		fs.Stats.Checking(src.Remote())
+		defer fs.Stats.DoneChecking(src.Remote())
+		if src.Size() != dst.Size() {
+			fs.Stats.Error()
+			fs.Errorf(src, "Sizes differ")
+			return true, false
+		}
+		cryptDst := dst.(*crypt.Object)
+		underlyingDst := cryptDst.UnWrap()
+		underlyingHash, err := underlyingDst.Hash(hashType)
+		if err != nil {
+			fs.Stats.Error()
+			fs.Errorf(dst, "Error reading hash from underlying %v: %v", underlyingDst, err)
+			return true, false
+		}
+		if underlyingHash == "" {
+			return false, true
+		}
+		cryptHash, err := fcrypt.ComputeHash(cryptDst, src, hashType)
+		if err != nil {
+			fs.Stats.Error()
+			fs.Errorf(dst, "Error computing hash: %v", err)
+			return true, false
+		}
+		if cryptHash == "" {
+			return false, true
+		}
+		if cryptHash != underlyingHash {
+			fs.Stats.Error()
+			fs.Errorf(src, "hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)
+			return true, false
+		}
+		fs.Debugf(src, "OK")
+		return false, false
+	}
+
+	return fs.CheckFn(fcrypt, fsrc, checkIdentical)
+}

crypt/cipher.go

@@ -416,7 +416,7 @@ type encrypter struct {
 }
 
 // newEncrypter creates a new file handle encrypting on the fly
-func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) {
+func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 	fh := &encrypter{
 		in:      in,
 		c:       c,
@@ -425,9 +425,13 @@ func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) {
 		bufSize: fileHeaderSize,
 	}
 	// Initialise nonce
-	err := fh.nonce.fromReader(c.cryptoRand)
+	if nonce != nil {
-	if err != nil {
+		fh.nonce = *nonce
-		return nil, err
+	} else {
+		err := fh.nonce.fromReader(c.cryptoRand)
+		if err != nil {
+			return nil, err
+		}
 	}
 	// Copy magic into buffer
 	copy(fh.buf, fileMagicBytes)
@@ -485,7 +489,7 @@ func (fh *encrypter) finish(err error) (int, error) {
 
 // Encrypt data encrypts the data stream
 func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
-	out, err := c.newEncrypter(in)
+	out, err := c.newEncrypter(in, nil)
 	if err != nil {
 		return nil, err
 	}

crypt/cipher_test.go

@@ -681,7 +681,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	c.cryptoRand = &zeroes{} // zero out the nonce
 	buf := make([]byte, bufSize)
 	source := newRandomSource(copySize)
-	encrypted, err := c.newEncrypter(source)
+	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
 	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
 	assert.NoError(t, err)
@@ -775,14 +775,14 @@ func TestNewEncrypter(t *testing.T) {
 
 	z := &zeroes{}
 
-	fh, err := c.newEncrypter(z)
+	fh, err := c.newEncrypter(z, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
 	assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
 
 	// Test error path
 	c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
-	fh, err = c.newEncrypter(z)
+	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
 	assert.Error(t, err, "short read of nonce")
 
@@ -795,7 +795,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	assert.NoError(t, err)
 
 	in := &errorReader{io.ErrUnexpectedEOF}
-	fh, err := c.newEncrypter(in)
+	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
 	n, err := io.CopyN(ioutil.Discard, fh, 1E6)

crypt/crypt.go

@@ -312,6 +312,59 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
+// ComputeHash takes the nonce from o, and encrypts the contents of
+// src with it, and calcuates the hash given by HashType on the fly
+//
+// Note that we break lots of encapsulation in this function.
+func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType fs.HashType) (hash string, err error) {
+	// Read the nonce - opening the file is sufficient to read the nonce in
+	in, err := o.Open()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read nonce")
+	}
+	nonce := in.(*decrypter).nonce
+	// fs.Debugf(o, "Read nonce % 2x", nonce)
+
+	// Check nonce isn't all zeros
+	isZero := true
+	for i := range nonce {
+		if nonce[i] != 0 {
+			isZero = false
+		}
+	}
+	if isZero {
+		fs.Errorf(o, "empty nonce read")
+	}
+
+	// Close in once we have read the nonce
+	err = in.Close()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to close nonce read")
+	}
+
+	// Open the src for input
+	in, err = src.Open()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to open src")
+	}
+	defer fs.CheckClose(in, &err)
+
+	// Now encrypt the src with the nonce
+	out, err := f.cipher.(*cipher).newEncrypter(in, &nonce)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make encrypter")
+	}
+
+	// pipe into hash
+	m := fs.NewMultiHasher()
+	_, err = io.Copy(m, out)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to hash data")
+	}
+
+	return m.Sums()[hashType], nil
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -366,6 +419,11 @@ func (o *Object) Hash(hash fs.HashType) (string, error) {
 	return "", nil
 }
 
+// UnWrap returns the wrapped Object
+func (o *Object) UnWrap() fs.Object {
+	return o.Object
+}
+
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
 	var offset int64

fs/operations.go

@@ -695,8 +695,14 @@ func checkIdentical(dst, src Object) (differ bool, noHash bool) {
 	return false, false
 }
 
-// Check the files in fsrc and fdst according to Size and hash
+// CheckFn checks the files in fsrc and fdst according to Size and
-func Check(fdst, fsrc Fs) error {
+// hash using checkFunction on each file to check the hashes.
+//
+// checkFunction sees if dst and src are identical
+//
+// it returns true if differences were found
+// it also returns whether it couldn't be hashed
+func CheckFn(fdst, fsrc Fs, checkFunction func(a, b Object) (differ bool, noHash bool)) error {
 	dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false, "")
 	if err != nil {
 		return err
@@ -709,10 +715,10 @@ func Check(fdst, fsrc Fs) error {
 
 	// Move all the common files into commonFiles and delete then
 	// from srcFiles and dstFiles
-	commonFiles := make(map[string][]Object)
+	commonFiles := make(map[string][2]Object)
 	for remote, src := range srcFiles {
 		if dst, ok := dstFiles[remote]; ok {
-			commonFiles[remote] = []Object{dst, src}
+			commonFiles[remote] = [2]Object{dst, src}
 			delete(srcFiles, remote)
 			delete(dstFiles, remote)
 		}
@@ -732,7 +738,7 @@ func Check(fdst, fsrc Fs) error {
 		atomic.AddInt32(&differences, 1)
 	}
 
-	checks := make(chan []Object, Config.Transfers)
+	checks := make(chan [2]Object, Config.Transfers)
 	go func() {
 		for _, check := range commonFiles {
 			checks <- check
@@ -746,7 +752,7 @@ func Check(fdst, fsrc Fs) error {
 		go func() {
 			defer checkerWg.Done()
 			for check := range checks {
-				differ, noHash := checkIdentical(check[0], check[1])
+				differ, noHash := checkFunction(check[0], check[1])
 				if differ {
 					atomic.AddInt32(&differences, 1)
 				}
@@ -769,6 +775,11 @@ func Check(fdst, fsrc Fs) error {
 	return nil
 }
 
+// Check the files in fsrc and fdst according to Size and hash
+func Check(fdst, fsrc Fs) error {
+	return CheckFn(fdst, fsrc, checkIdentical)
+}
+
 // ListFn lists the Fs to the supplied function
 //
 // Lists in parallel which may get them out of order