From 01c747e7db5fae2014800a8b67d38ecdafdfa6d5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 12 Feb 2017 16:30:18 +0000 Subject: [PATCH] Add cryptcheck command to check integrity of crypt remotes #1102 --- cmd/all/all.go | 1 + cmd/cryptcheck/cryptcheck.go | 108 +++++++++++++++++++++++++++++++++++ crypt/cipher.go | 14 +++-- crypt/cipher_test.go | 8 +-- crypt/crypt.go | 58 +++++++++++++++++++ fs/operations.go | 23 ++++++-- 6 files changed, 197 insertions(+), 15 deletions(-) create mode 100644 cmd/cryptcheck/cryptcheck.go diff --git a/cmd/all/all.go b/cmd/all/all.go index 9a256e825..2057d2930 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -11,6 +11,7 @@ import ( _ "github.com/ncw/rclone/cmd/config" _ "github.com/ncw/rclone/cmd/copy" _ "github.com/ncw/rclone/cmd/copyto" + _ "github.com/ncw/rclone/cmd/cryptcheck" _ "github.com/ncw/rclone/cmd/dedupe" _ "github.com/ncw/rclone/cmd/delete" _ "github.com/ncw/rclone/cmd/genautocomplete" diff --git a/cmd/cryptcheck/cryptcheck.go b/cmd/cryptcheck/cryptcheck.go new file mode 100644 index 000000000..c3f89c321 --- /dev/null +++ b/cmd/cryptcheck/cryptcheck.go @@ -0,0 +1,108 @@ +package cryptcheck + +import ( + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/crypt" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +func init() { + cmd.Root.AddCommand(commandDefintion) +} + +var commandDefintion = &cobra.Command{ + Use: "cryptcheck remote:path cryptedremote:path", + Short: `Cryptcheck checks the integritity of a crypted remote.`, + Long: ` +rclone cryptcheck checks a remote against a crypted remote. This is +the equivalent of running rclone check, but able to check the +checksums of the crypted remote. + +For it to work the underlying remote of the cryptedremote must support +some kind of checksum. + +It works by reading the nonce from each file on the cryptedremote: and +using that to encrypt each file on the remote:. It then checks the +checksum of the underlying file on the cryptedremote: against the +checksum of the file it has just encrypted. + +Use it like this + + rclone cryptcheck /path/to/files encryptedremote:path + +You can use it like this also, but that will involve downloading all +the files in remote:path. + + rclone cryptcheck remote:path encryptedremote:path + +After it has run it will log the status of the encryptedremote:. +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(2, 2, command, args) + fsrc, fdst := cmd.NewFsSrcDst(args) + cmd.Run(false, true, command, func() error { + return cryptCheck(fdst, fsrc) + }) + }, +} + +// cryptCheck checks the integrity of a crypted remote +func cryptCheck(fdst, fsrc fs.Fs) error { + // Check to see fcrypt is a crypt + fcrypt, ok := fdst.(*crypt.Fs) + if !ok { + return errors.Errorf("%s:%s is not a crypt remote", fdst.Name(), fdst.Root()) + } + // Find a hash to use + funderlying := fcrypt.UnWrap() + hashType := funderlying.Hashes().GetOne() + if hashType == fs.HashNone { + return errors.Errorf("%s:%s does not support any hashes", funderlying.Name(), funderlying.Root()) + } + fs.Infof(nil, "Using %v for hash comparisons", hashType) + + // checkIdentical checks to see if dst and src are identical + // + // it returns true if differences were found + // it also returns whether it couldn't be hashed + checkIdentical := func(dst, src fs.Object) (differ bool, noHash bool) { + fs.Stats.Checking(src.Remote()) + defer fs.Stats.DoneChecking(src.Remote()) + if src.Size() != dst.Size() { + fs.Stats.Error() + fs.Errorf(src, "Sizes differ") + return true, false + } + cryptDst := dst.(*crypt.Object) + underlyingDst := cryptDst.UnWrap() + underlyingHash, err := underlyingDst.Hash(hashType) + if err != nil { + fs.Stats.Error() + fs.Errorf(dst, "Error reading hash from underlying %v: %v", underlyingDst, err) + return true, false + } + if underlyingHash == "" { + return false, true + } + cryptHash, err := fcrypt.ComputeHash(cryptDst, src, hashType) + if err != nil { + fs.Stats.Error() + fs.Errorf(dst, "Error computing hash: %v", err) + return true, false + } + if cryptHash == "" { + return false, true + } + if cryptHash != underlyingHash { + fs.Stats.Error() + fs.Errorf(src, "hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash) + return true, false + } + fs.Debugf(src, "OK") + return false, false + } + + return fs.CheckFn(fcrypt, fsrc, checkIdentical) +} diff --git a/crypt/cipher.go b/crypt/cipher.go index cd3cd9d3a..1be758eae 100644 --- a/crypt/cipher.go +++ b/crypt/cipher.go @@ -416,7 +416,7 @@ type encrypter struct { } // newEncrypter creates a new file handle encrypting on the fly -func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) { +func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) { fh := &encrypter{ in: in, c: c, @@ -425,9 +425,13 @@ func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) { bufSize: fileHeaderSize, } // Initialise nonce - err := fh.nonce.fromReader(c.cryptoRand) - if err != nil { - return nil, err + if nonce != nil { + fh.nonce = *nonce + } else { + err := fh.nonce.fromReader(c.cryptoRand) + if err != nil { + return nil, err + } } // Copy magic into buffer copy(fh.buf, fileMagicBytes) @@ -485,7 +489,7 @@ func (fh *encrypter) finish(err error) (int, error) { // Encrypt data encrypts the data stream func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) { - out, err := c.newEncrypter(in) + out, err := c.newEncrypter(in, nil) if err != nil { return nil, err } diff --git a/crypt/cipher_test.go b/crypt/cipher_test.go index 2face09fa..b745a8381 100644 --- a/crypt/cipher_test.go +++ b/crypt/cipher_test.go @@ -681,7 +681,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) { c.cryptoRand = &zeroes{} // zero out the nonce buf := make([]byte, bufSize) source := newRandomSource(copySize) - encrypted, err := c.newEncrypter(source) + encrypted, err := c.newEncrypter(source, nil) assert.NoError(t, err) decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted)) assert.NoError(t, err) @@ -775,14 +775,14 @@ func TestNewEncrypter(t *testing.T) { z := &zeroes{} - fh, err := c.newEncrypter(z) + fh, err := c.newEncrypter(z, nil) assert.NoError(t, err) assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce) assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32]) // Test error path c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn") - fh, err = c.newEncrypter(z) + fh, err = c.newEncrypter(z, nil) assert.Nil(t, fh) assert.Error(t, err, "short read of nonce") @@ -795,7 +795,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) { assert.NoError(t, err) in := &errorReader{io.ErrUnexpectedEOF} - fh, err := c.newEncrypter(in) + fh, err := c.newEncrypter(in, nil) assert.NoError(t, err) n, err := io.CopyN(ioutil.Discard, fh, 1E6) diff --git a/crypt/crypt.go b/crypt/crypt.go index b2369dca4..cb1a8f9ee 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -312,6 +312,59 @@ func (f *Fs) UnWrap() fs.Fs { return f.Fs } +// ComputeHash takes the nonce from o, and encrypts the contents of +// src with it, and calcuates the hash given by HashType on the fly +// +// Note that we break lots of encapsulation in this function. +func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType fs.HashType) (hash string, err error) { + // Read the nonce - opening the file is sufficient to read the nonce in + in, err := o.Open() + if err != nil { + return "", errors.Wrap(err, "failed to read nonce") + } + nonce := in.(*decrypter).nonce + // fs.Debugf(o, "Read nonce % 2x", nonce) + + // Check nonce isn't all zeros + isZero := true + for i := range nonce { + if nonce[i] != 0 { + isZero = false + } + } + if isZero { + fs.Errorf(o, "empty nonce read") + } + + // Close in once we have read the nonce + err = in.Close() + if err != nil { + return "", errors.Wrap(err, "failed to close nonce read") + } + + // Open the src for input + in, err = src.Open() + if err != nil { + return "", errors.Wrap(err, "failed to open src") + } + defer fs.CheckClose(in, &err) + + // Now encrypt the src with the nonce + out, err := f.cipher.(*cipher).newEncrypter(in, &nonce) + if err != nil { + return "", errors.Wrap(err, "failed to make encrypter") + } + + // pipe into hash + m := fs.NewMultiHasher() + _, err = io.Copy(m, out) + if err != nil { + return "", errors.Wrap(err, "failed to hash data") + } + + return m.Sums()[hashType], nil +} + // Object describes a wrapped for being read from the Fs // // This decrypts the remote name and decrypts the data @@ -366,6 +419,11 @@ func (o *Object) Hash(hash fs.HashType) (string, error) { return "", nil } +// UnWrap returns the wrapped Object +func (o *Object) UnWrap() fs.Object { + return o.Object +} + // Open opens the file for read. Call Close() on the returned io.ReadCloser func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) { var offset int64 diff --git a/fs/operations.go b/fs/operations.go index 9be889f57..6a149b4f6 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -695,8 +695,14 @@ func checkIdentical(dst, src Object) (differ bool, noHash bool) { return false, false } -// Check the files in fsrc and fdst according to Size and hash -func Check(fdst, fsrc Fs) error { +// CheckFn checks the files in fsrc and fdst according to Size and +// hash using checkFunction on each file to check the hashes. +// +// checkFunction sees if dst and src are identical +// +// it returns true if differences were found +// it also returns whether it couldn't be hashed +func CheckFn(fdst, fsrc Fs, checkFunction func(a, b Object) (differ bool, noHash bool)) error { dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false, "") if err != nil { return err @@ -709,10 +715,10 @@ func Check(fdst, fsrc Fs) error { // Move all the common files into commonFiles and delete then // from srcFiles and dstFiles - commonFiles := make(map[string][]Object) + commonFiles := make(map[string][2]Object) for remote, src := range srcFiles { if dst, ok := dstFiles[remote]; ok { - commonFiles[remote] = []Object{dst, src} + commonFiles[remote] = [2]Object{dst, src} delete(srcFiles, remote) delete(dstFiles, remote) } @@ -732,7 +738,7 @@ func Check(fdst, fsrc Fs) error { atomic.AddInt32(&differences, 1) } - checks := make(chan []Object, Config.Transfers) + checks := make(chan [2]Object, Config.Transfers) go func() { for _, check := range commonFiles { checks <- check @@ -746,7 +752,7 @@ func Check(fdst, fsrc Fs) error { go func() { defer checkerWg.Done() for check := range checks { - differ, noHash := checkIdentical(check[0], check[1]) + differ, noHash := checkFunction(check[0], check[1]) if differ { atomic.AddInt32(&differences, 1) } @@ -769,6 +775,11 @@ func Check(fdst, fsrc Fs) error { return nil } +// Check the files in fsrc and fdst according to Size and hash +func Check(fdst, fsrc Fs) error { + return CheckFn(fdst, fsrc, checkIdentical) +} + // ListFn lists the Fs to the supplied function // // Lists in parallel which may get them out of order