From eba0a3633b5c2a68e2210d54e3a25fc67a9a0234 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 20 Oct 2016 17:47:33 +0100 Subject: [PATCH] crypt: speed up repeated seeking - fixes #804 --- cmd/mount/read.go | 31 ++++++---- crypt/cipher.go | 133 +++++++++++++++++++++++++++++++++++++------ crypt/cipher_test.go | 51 +++++++++++++++++ crypt/crypt.go | 48 +++------------- 4 files changed, 196 insertions(+), 67 deletions(-) diff --git a/cmd/mount/read.go b/cmd/mount/read.go index 2f36a0c03..f3b8ca993 100644 --- a/cmd/mount/read.go +++ b/cmd/mount/read.go @@ -41,17 +41,28 @@ var _ fusefs.HandleReader = (*ReadFileHandle)(nil) // seek to a new offset func (fh *ReadFileHandle) seek(offset int64) error { - fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset) - r, err := fh.o.Open(&fs.SeekOption{Offset: offset}) - if err != nil { - fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err) - return err + // Can we seek it directly? + if do, ok := fh.r.(io.Seeker); ok { + fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d (io.Seeker)", fh.offset, offset) + _, err := do.Seek(offset, io.SeekStart) + if err != nil { + fs.Debug(fh.o, "ReadFileHandle.Read io.Seeker failed: %v", err) + return err + } + } else { + fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset) + // if not re-open with a seek + r, err := fh.o.Open(&fs.SeekOption{Offset: offset}) + if err != nil { + fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err) + return err + } + err = fh.r.Close() + if err != nil { + fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err) + } + fh.r = r } - err = fh.r.Close() - if err != nil { - fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err) - } - fh.r = r fh.offset = offset return nil } diff --git a/crypt/cipher.go b/crypt/cipher.go index 102144895..3b92d81d8 100644 --- a/crypt/cipher.go +++ b/crypt/cipher.go @@ -8,6 +8,7 @@ import ( "encoding/base32" "fmt" "io" + "io/ioutil" "strings" "sync" "unicode/utf8" @@ -21,7 +22,7 @@ import ( "github.com/rfjakob/eme" ) -// Constancs +// Constants const ( nameCipherBlockSize = aes.BlockSize fileMagic = "RCLONE\x00\x00" @@ -55,6 +56,16 @@ var ( fileMagicBytes = []byte(fileMagic) ) +// ReadSeekCloser is the interface of the read handles +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer +} + +// OpenAtOffset opens the file handle at the offset given +type OpenAtOffset func(offset int64) (io.ReadCloser, error) + // Cipher is used to swap out the encryption implementations type Cipher interface { // EncryptFileName encrypts a file path @@ -69,6 +80,8 @@ type Cipher interface { EncryptData(io.Reader) (io.Reader, error) // DecryptData DecryptData(io.ReadCloser) (io.ReadCloser, error) + // DecryptDataSeek decrypt at a given position + DecryptDataSeek(open OpenAtOffset, offset int64) (ReadSeekCloser, error) // EncryptedSize calculates the size of the data when encrypted EncryptedSize(int64) int64 // DecryptedSize calculates the size of the data when decrypted @@ -476,14 +489,16 @@ func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) { // decrypter decrypts an io.ReaderCloser on the fly type decrypter struct { - rc io.ReadCloser - nonce nonce - c *cipher - buf []byte - readBuf []byte - bufIndex int - bufSize int - err error + rc io.ReadCloser + nonce nonce + initialNonce nonce + c *cipher + buf []byte + readBuf []byte + bufIndex int + bufSize int + err error + open OpenAtOffset } // newDecrypter creates a new file handle decrypting on the fly @@ -509,6 +524,30 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) { } // retreive the nonce fh.nonce.fromBuf(readBuf[fileMagicSize:]) + fh.initialNonce = fh.nonce + return fh, nil +} + +// newDecrypterSeek creates a new file handle decrypting on the fly +func (c *cipher) newDecrypterSeek(open OpenAtOffset, offset int64) (fh *decrypter, err error) { + // Open initially with no seek + rc, err := open(0) + if err != nil { + return nil, err + } + // Open the stream which fills in the nonce + fh, err = c.newDecrypter(rc) + if err != nil { + return nil, err + } + fh.open = open // will be called by fh.Seek + if offset != 0 { + _, err = fh.Seek(offset, io.SeekStart) + if err != nil { + _ = fh.Close() + return nil, err + } + } return fh, nil } @@ -549,15 +588,60 @@ func (fh *decrypter) Read(p []byte) (n int, err error) { return n, nil } -// seek the decryption forwards the amount given -// -// returns an offset for the underlying rc to be seeked and the number -// of bytes to be discarded -func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) { - blocks, discard := offset/blockDataSize, offset%blockDataSize - underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) - fh.nonce.add(uint64(blocks)) - return +// Seek as per io.Seeker +func (fh *decrypter) Seek(offset int64, whence int) (int64, error) { + if fh.open == nil { + return 0, fh.finish(errors.New("can't seek - not initialised with newDecrypterSeek")) + } + if whence != io.SeekStart { + return 0, fh.finish(errors.New("can only seek from the start")) + } + + // Reset error or return it if not EOF + if fh.err == io.EOF { + fh.err = nil + } else if fh.err != nil { + return 0, fh.err + } + + // Can we seek it directly? + if do, ok := fh.rc.(io.Seeker); ok { + _, err := do.Seek(offset, io.SeekStart) + if err != nil { + return 0, fh.finish(err) + } + } else { + // if not reopen with seek + _ = fh.rc.Close() // close underlying file + fh.rc = nil + + // blocks we need to seek, plus bytes we need to discard + blocks, discard := offset/blockDataSize, offset%blockDataSize + + // Offset in underlying stream we need to seek + underlyingOffset := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) + + // Move the nonce on the correct number of blocks from the start + fh.nonce = fh.initialNonce + fh.nonce.add(uint64(blocks)) + + // Re-open the underlying object with the offset given + rc, err := fh.open(underlyingOffset) + if err != nil { + return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset")) + } + + // Set the file handle + fh.rc = rc + + // Discard excess bytes + _, err = io.CopyN(ioutil.Discard, fh, discard) + if err != nil { + return 0, fh.finish(err) + } + } + + return offset, nil } // finish sets the final error and tidies up @@ -604,6 +688,19 @@ func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) { return out, nil } +// DecryptDataSeek decrypts the data stream from offset +// +// The open function must return a ReadCloser opened to the offset supplied +// +// You must use this form of DecryptData if you might want to Seek the file handle +func (c *cipher) DecryptDataSeek(open OpenAtOffset, offset int64) (ReadSeekCloser, error) { + out, err := c.newDecrypterSeek(open, offset) + if err != nil { + return nil, err + } + return out, nil +} + // EncryptedSize calculates the size of the data when encrypted func (c *cipher) EncryptedSize(size int64) int64 { blocks, residue := size/blockDataSize, size%blockDataSize diff --git a/crypt/cipher_test.go b/crypt/cipher_test.go index 405c04870..76dcd1133 100644 --- a/crypt/cipher_test.go +++ b/crypt/cipher_test.go @@ -854,6 +854,57 @@ func TestNewDecrypter(t *testing.T) { } } +func TestNewDecrypterSeek(t *testing.T) { + c, err := newCipher(NameEncryptionStandard, "", "") + assert.NoError(t, err) + c.cryptoRand = &zeroes{} // nodge the crypto rand generator + + // Make random data + const dataSize = 150000 + plaintext, err := ioutil.ReadAll(newRandomSource(dataSize)) + assert.NoError(t, err) + + // Encrypt the data + buf := bytes.NewBuffer(plaintext) + encrypted, err := c.EncryptData(buf) + assert.NoError(t, err) + ciphertext, err := ioutil.ReadAll(encrypted) + assert.NoError(t, err) + + trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, + 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, + 4095, 4096, 4097, 8191, 8192, 8193, 16383, 16384, 16385, 32767, 32768, 32769, + 65535, 65536, 65537, 131071, 131072, 131073, dataSize - 1, dataSize} + + // Open stream with a seek of underlyingOffset + open := func(underlyingOffset int64) (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):])), nil + } + + // Now try decoding it with a open/seek + for _, offset := range trials { + rc, err := c.DecryptDataSeek(open, int64(offset)) + assert.NoError(t, err) + + seekedDecrypted, err := ioutil.ReadAll(rc) + assert.NoError(t, err) + + assert.Equal(t, plaintext[offset:], seekedDecrypted) + } + + // Now try decoding it with a single open and lots of seeks + rc, err := c.DecryptDataSeek(open, 0) + for _, offset := range trials { + _, err := rc.Seek(int64(offset), io.SeekStart) + assert.NoError(t, err) + + seekedDecrypted, err := ioutil.ReadAll(rc) + assert.NoError(t, err) + + assert.Equal(t, plaintext[offset:], seekedDecrypted) + } +} + func TestDecrypterRead(t *testing.T) { c, err := newCipher(NameEncryptionStandard, "", "") assert.NoError(t, err) diff --git a/crypt/crypt.go b/crypt/crypt.go index 57d68964f..18c379163 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -4,7 +4,6 @@ package crypt import ( "fmt" "io" - "io/ioutil" "path" "sync" @@ -298,7 +297,7 @@ func (o *Object) Hash(hash fs.HashType) (string, error) { } // Open opens the file for read. Call Close() on the returned io.ReadCloser -func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { +func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) { var offset int64 for _, option := range options { switch x := option.(type) { @@ -310,46 +309,17 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { } } } - in, err := o.Object.Open() + rc, err = o.f.cipher.DecryptDataSeek(func(underlyingOffset int64) (io.ReadCloser, error) { + if underlyingOffset == 0 { + // Open with no seek + return o.Object.Open() + } + // Open stream with a seek of underlyingOffset + return o.Object.Open(&fs.SeekOption{Offset: underlyingOffset}) + }, offset) if err != nil { return nil, err } - - // This reads the header and checks it is OK - rc, err := o.f.cipher.DecryptData(in) - if err != nil { - return nil, err - } - - // If seeking required, then... - if offset != 0 { - // FIXME could cache the unseeked decrypter as we re-read the header on every seek - decrypter := rc.(*decrypter) - - // Seek the decrypter and work out where to seek the - // underlying file and how many bytes to discard - underlyingOffset, discard := decrypter.seek(offset) - - // Re-open stream with a seek of underlyingOffset - err = in.Close() - if err != nil { - return nil, err - } - in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset}) - if err != nil { - return nil, err - } - - // Update the stream - decrypter.rc = in - - // Discard the bytes - _, err = io.CopyN(ioutil.Discard, decrypter, discard) - if err != nil { - return nil, err - } - } - return rc, err }