KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

Implement crypt for encrypted remotes - #219

Browse source
This commit is contained in:
Nick Craig-Wood 2016-07-25 19:18:56 +01:00
parent b4b4b6cb1c
commit 226c2a0d83
10 changed files with 2333 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

574
crypt/cipher.go Normal file
View file

@ -0,0 +1,574 @@
package crypt
import (
"bytes"
"crypto/aes"
gocipher "crypto/cipher"
"crypto/rand"
"encoding/base32"
"io"
"strings"
"sync"
"unicode/utf8"
"github.com/ncw/rclone/crypt/pkcs7"
"github.com/pkg/errors"
"golang.org/x/crypto/nacl/secretbox"
"golang.org/x/crypto/scrypt"
"github.com/rfjakob/eme"
)
// Constancs
const (
nameCipherBlockSize = aes.BlockSize
fileMagic = "RCLONE\x00\x00"
fileMagicSize = len(fileMagic)
fileNonceSize = 24
fileHeaderSize = fileMagicSize + fileNonceSize
blockHeaderSize = secretbox.Overhead
blockDataSize = 64 * 1024
blockSize = blockHeaderSize + blockDataSize
)
// Errors returned by cipher
var (
ErrorBadDecryptUTF8 = errors.New("bad decryption - utf-8 invalid")
ErrorBadDecryptControlChar = errors.New("bad decryption - contains control chars")
ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize")
ErrorTooShortAfterDecode = errors.New("too short after base32 decode")
ErrorEncryptedFileTooShort = errors.New("file is too short to be encrypted")
ErrorEncryptedFileBadHeader = errors.New("file has truncated block header")
ErrorEncryptedBadMagic = errors.New("not an encrypted file - bad magic string")
ErrorEncryptedBadBlock = errors.New("failed to authenticate decrypted block - bad password?")
ErrorBadBase32Encoding = errors.New("bad base32 filename encoding")
ErrorBadSpreadNotSingleChar = errors.New("bad unspread - not single character")
ErrorBadSpreadResultTooShort = errors.New("bad unspread - result too short")
ErrorBadSpreadDidntMatch = errors.New("bad unspread - directory prefix didn't match")
ErrorFileClosed = errors.New("file already closed")
scryptSalt = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
)
// Global variables
var (
fileMagicBytes = []byte(fileMagic)
)
// Cipher is used to swap out the encryption implementations
type Cipher interface {
// EncryptName encrypts a file path
EncryptName(string) string
// DecryptName decrypts a file path, returns error if decrypt was invalid
DecryptName(string) (string, error)
// EncryptData
EncryptData(io.Reader) (io.Reader, error)
// DecryptData
DecryptData(io.ReadCloser) (io.ReadCloser, error)
// EncryptedSize calculates the size of the data when encrypted
EncryptedSize(int64) int64
// DecryptedSize calculates the size of the data when decrypted
DecryptedSize(int64) (int64, error)
}
type cipher struct {
dataKey [32]byte // Key for secretbox
nameKey [32]byte // 16,24 or 32 bytes
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
block gocipher.Block
flatten int // set flattening level - 0 is off
buffers sync.Pool // encrypt/decrypt buffers
cryptoRand io.Reader // read crypto random numbers from here
}
func newCipher(flatten int, password string) (*cipher, error) {
c := &cipher{
flatten: flatten,
cryptoRand: rand.Reader,
}
c.buffers.New = func() interface{} {
return make([]byte, blockSize)
}
err := c.Key(password)
if err != nil {
return nil, err
}
return c, nil
}
// Key creates all the internal keys from the password passed in using
// scrypt. We use a fixed salt just to make attackers lives slighty
// harder than using no salt.
//
// Note that empty passsword makes all 0x00 keys which is used in the
// tests.
func (c *cipher) Key(password string) (err error) {
const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
var key []byte
if password == "" {
key = make([]byte, keySize)
} else {
key, err = scrypt.Key([]byte(password), scryptSalt, 16384, 8, 1, keySize)
if err != nil {
return err
}
}
copy(c.dataKey[:], key)
copy(c.nameKey[:], key[len(c.dataKey):])
copy(c.nameTweak[:], key[len(c.dataKey)+len(c.nameKey):])
// Key the name cipher
c.block, err = aes.NewCipher(c.nameKey[:])
return err
}
// getBlock gets a block from the pool of size blockSize
func (c *cipher) getBlock() []byte {
return c.buffers.Get().([]byte)
}
// putBlock returns a block to the pool of size blockSize
func (c *cipher) putBlock(buf []byte) {
if len(buf) != blockSize {
panic("bad blocksize returned to pool")
}
c.buffers.Put(buf)
}
// check to see if the byte string is valid with no control characters
// from 0x00 to 0x1F and is a valid UTF-8 string
func checkValidString(buf []byte) error {
for i := range buf {
c := buf[i]
if c >= 0x00 && c < 0x20 || c == 0x7F {
return ErrorBadDecryptControlChar
}
}
if !utf8.Valid(buf) {
return ErrorBadDecryptUTF8
}
return nil
}
// encodeFileName encodes a filename using a modified version of
// standard base32 as described in RFC4648
//
// The standard encoding is modified in two ways
// * it becomes lower case (no-one likes upper case filenames!)
// * we strip the padding character `=`
func encodeFileName(in []byte) string {
encoded := base32.HexEncoding.EncodeToString(in)
encoded = strings.TrimRight(encoded, "=")
return strings.ToLower(encoded)
}
// decodeFileName decodes a filename as encoded by encodeFileName
func decodeFileName(in string) ([]byte, error) {
if strings.HasSuffix(in, "=") {
return nil, ErrorBadBase32Encoding
}
// First figure out how many padding characters to add
roundUpToMultipleOf8 := (len(in) + 7) &^ 7
equals := roundUpToMultipleOf8 - len(in)
in = strings.ToUpper(in) + "========"[:equals]
return base32.HexEncoding.DecodeString(in)
}
// encryptSegment encrypts a path segment
//
// This uses EME with AES
//
// EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
// 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
// Rogaway.
//
// This makes for determinstic encryption which is what we want - the
// same filename must encrypt to the same thing.
//
// This means that
// * filenames with the same name will encrypt the same
// * filenames which start the same won't have a common prefix
func (c *cipher) encryptSegment(plaintext string) string {
if plaintext == "" {
return ""
}
paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext))
ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt)
return encodeFileName(ciphertext)
}
// decryptSegment decrypts a path segment
func (c *cipher) decryptSegment(ciphertext string) (string, error) {
if ciphertext == "" {
return "", nil
}
rawCiphertext, err := decodeFileName(ciphertext)
if err != nil {
return "", err
}
if len(rawCiphertext)%nameCipherBlockSize != 0 {
return "", ErrorNotAMultipleOfBlocksize
}
if len(rawCiphertext) == 0 {
// not possible if decodeFilename() working correctly
return "", ErrorTooShortAfterDecode
}
paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt)
plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext)
if err != nil {
return "", err
}
err = checkValidString(plaintext)
if err != nil {
return "", err
}
return string(plaintext), err
}
// spread a name over the given number of directory levels
//
// if in isn't long enough dirs will be reduces
func spreadName(dirs int, in string) string {
if dirs > len(in) {
dirs = len(in)
}
prefix := ""
for i := 0; i < dirs; i++ {
prefix += string(in[i]) + "/"
}
return prefix + in
}
// reverse spreadName, returning an error if not in spread format
//
// This decodes any level of spreading
func unspreadName(in string) (string, error) {
in = strings.ToLower(in)
segments := strings.Split(in, "/")
if len(segments) == 0 {
return in, nil
}
out := segments[len(segments)-1]
segments = segments[:len(segments)-1]
for i, s := range segments {
if len(s) != 1 {
return "", ErrorBadSpreadNotSingleChar
}
if i >= len(out) {
return "", ErrorBadSpreadResultTooShort
}
if s[0] != out[i] {
return "", ErrorBadSpreadDidntMatch
}
}
return out, nil
}
// EncryptName encrypts a file path
func (c *cipher) EncryptName(in string) string {
if c.flatten > 0 {
return spreadName(c.flatten, c.encryptSegment(in))
}
segments := strings.Split(in, "/")
for i := range segments {
segments[i] = c.encryptSegment(segments[i])
}
return strings.Join(segments, "/")
}
// DecryptName decrypts a file path
func (c *cipher) DecryptName(in string) (string, error) {
if c.flatten > 0 {
unspread, err := unspreadName(in)
if err != nil {
return "", err
}
return c.decryptSegment(unspread)
}
segments := strings.Split(in, "/")
for i := range segments {
var err error
segments[i], err = c.decryptSegment(segments[i])
if err != nil {
return "", err
}
}
return strings.Join(segments, "/"), nil
}
// nonce is an NACL secretbox nonce
type nonce [fileNonceSize]byte
// pointer returns the nonce as a *[24]byte for secretbox
func (n *nonce) pointer() *[fileNonceSize]byte {
return (*[fileNonceSize]byte)(n)
}
// fromReader fills the nonce from an io.Reader - normally the OSes
// crypto random number generator
func (n *nonce) fromReader(in io.Reader) error {
read, err := io.ReadFull(in, (*n)[:])
if read != fileNonceSize {
return errors.Wrap(err, "short read of nonce")
}
return nil
}
// fromBuf fills the nonce from the buffer passed in
func (n *nonce) fromBuf(buf []byte) {
read := copy((*n)[:], buf)
if read != fileNonceSize {
panic("buffer to short to read nonce")
}
}
// increment to add 1 to the nonce
func (n *nonce) increment() {
for i := 0; i < len(*n); i++ {
digit := (*n)[i]
newDigit := digit + 1
(*n)[i] = newDigit
if newDigit >= digit {
// exit if no carry
break
}
}
}
// encrypter encrypts an io.Reader on the fly
type encrypter struct {
in io.Reader
c *cipher
nonce nonce
buf []byte
readBuf []byte
bufIndex int
bufSize int
err error
}
// newEncrypter creates a new file handle encrypting on the fly
func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) {
fh := &encrypter{
in: in,
c: c,
buf: c.getBlock(),
readBuf: c.getBlock(),
bufSize: fileHeaderSize,
}
// Initialise nonce
err := fh.nonce.fromReader(c.cryptoRand)
if err != nil {
return nil, err
}
// Copy magic into buffer
copy(fh.buf, fileMagicBytes)
// Copy nonce into buffer
copy(fh.buf[fileMagicSize:], fh.nonce[:])
return fh, nil
}
// Read as per io.Reader
func (fh *encrypter) Read(p []byte) (n int, err error) {
if fh.err != nil {
return 0, fh.err
}
if fh.bufIndex >= fh.bufSize {
// Read data
// FIXME should overlap the reads with a go-routine and 2 buffers?
readBuf := fh.readBuf[:blockDataSize]
n, err = io.ReadFull(fh.in, readBuf)
if err == io.EOF {
// ReadFull only returns n=0 and EOF
return fh.finish(io.EOF)
} else if err == io.ErrUnexpectedEOF {
// Next read will return EOF
} else if err != nil {
return fh.finish(err)
}
// Write nonce to start of block
copy(fh.buf, fh.nonce[:])
// Encrypt the block using the nonce
block := fh.buf
secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
fh.bufIndex = 0
fh.bufSize = blockHeaderSize + n
fh.nonce.increment()
}
n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
fh.bufIndex += n
return n, nil
}
// finish sets the final error and tidies up
func (fh *encrypter) finish(err error) (int, error) {
if fh.err != nil {
return 0, fh.err
}
fh.err = err
fh.c.putBlock(fh.buf)
fh.c.putBlock(fh.readBuf)
return 0, err
}
// Encrypt data encrypts the data stream
func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
out, err := c.newEncrypter(in)
if err != nil {
return nil, err
}
return out, nil
}
// decrypter decrypts an io.ReaderCloser on the fly
type decrypter struct {
rc io.ReadCloser
nonce nonce
c *cipher
buf []byte
readBuf []byte
bufIndex int
bufSize int
err error
}
// newDecrypter creates a new file handle decrypting on the fly
func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
fh := &decrypter{
rc: rc,
c: c,
buf: c.getBlock(),
readBuf: c.getBlock(),
}
// Read file header (magic + nonce)
readBuf := fh.readBuf[:fileHeaderSize]
_, err := io.ReadFull(fh.rc, readBuf)
if err == io.EOF || err == io.ErrUnexpectedEOF {
// This read from 0..fileHeaderSize-1 bytes
return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
} else if err != nil {
return nil, fh.finishAndClose(err)
}
// check the magic
if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
}
// retreive the nonce
fh.nonce.fromBuf(readBuf[fileMagicSize:])
return fh, nil
}
// Read as per io.Reader
func (fh *decrypter) Read(p []byte) (n int, err error) {
if fh.err != nil {
return 0, fh.err
}
if fh.bufIndex >= fh.bufSize {
// Read data
// FIXME should overlap the reads with a go-routine and 2 buffers?
readBuf := fh.readBuf
n, err = io.ReadFull(fh.rc, readBuf)
if err == io.EOF {
// ReadFull only returns n=0 and EOF
return 0, fh.finish(io.EOF)
} else if err == io.ErrUnexpectedEOF {
// Next read will return EOF
} else if err != nil {
return 0, fh.finish(err)
}
// Check header + 1 byte exists
if n <= blockHeaderSize {
return 0, fh.finish(ErrorEncryptedFileBadHeader)
}
// Decrypt the block using the nonce
block := fh.buf
_, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
if !ok {
return 0, fh.finish(ErrorEncryptedBadBlock)
}
fh.bufIndex = 0
fh.bufSize = n - blockHeaderSize
fh.nonce.increment()
}
n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
fh.bufIndex += n
return n, nil
}
// finish sets the final error and tidies up
func (fh *decrypter) finish(err error) error {
if fh.err != nil {
return fh.err
}
fh.err = err
fh.c.putBlock(fh.buf)
fh.c.putBlock(fh.readBuf)
return err
}
// Close
func (fh *decrypter) Close() error {
// Check already closed
if fh.err == ErrorFileClosed {
return fh.err
}
// Closed before reading EOF so not finish()ed yet
if fh.err == nil {
_ = fh.finish(io.EOF)
}
// Show file now closed
fh.err = ErrorFileClosed
return fh.rc.Close()
}
// finishAndClose does finish then Close()
//
// Used when we are returning a nil fh from new
func (fh *decrypter) finishAndClose(err error) error {
_ = fh.finish(err)
_ = fh.Close()
return err
}
// DecryptData decrypts the data stream
func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
out, err := c.newDecrypter(rc)
if err != nil {
return nil, err
}
return out, nil
}
// EncryptedSize calculates the size of the data when encrypted
func (c *cipher) EncryptedSize(size int64) int64 {
blocks, residue := size/blockDataSize, size%blockDataSize
encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
if residue != 0 {
encryptedSize += blockHeaderSize + residue
}
return encryptedSize
}
// DecryptedSize calculates the size of the data when decrypted
func (c *cipher) DecryptedSize(size int64) (int64, error) {
size -= int64(fileHeaderSize)
if size < 0 {
return 0, ErrorEncryptedFileTooShort
}
blocks, residue := size/blockSize, size%blockSize
decryptedSize := blocks * blockDataSize
if residue != 0 {
residue -= blockHeaderSize
if residue <= 0 {
return 0, ErrorEncryptedFileBadHeader
}
}
decryptedSize += residue
return decryptedSize, nil
}
// check interfaces
var (
_ Cipher = (*cipher)(nil)
_ io.ReadCloser = (*decrypter)(nil)
_ io.Reader = (*encrypter)(nil)
)

816
crypt/cipher_test.go Normal file
View file

@ -0,0 +1,816 @@
package crypt
import (
"bytes"
"encoding/base32"
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
"github.com/ncw/rclone/crypt/pkcs7"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidString(t *testing.T) {
for _, test := range []struct {
in string
expected error
}{
{"", nil},
{"\x01", ErrorBadDecryptControlChar},
{"a\x02", ErrorBadDecryptControlChar},
{"abc\x03", ErrorBadDecryptControlChar},
{"abc\x04def", ErrorBadDecryptControlChar},
{"\x05d", ErrorBadDecryptControlChar},
{"\x06def", ErrorBadDecryptControlChar},
{"\x07", ErrorBadDecryptControlChar},
{"\x08", ErrorBadDecryptControlChar},
{"\x09", ErrorBadDecryptControlChar},
{"\x0A", ErrorBadDecryptControlChar},
{"\x0B", ErrorBadDecryptControlChar},
{"\x0C", ErrorBadDecryptControlChar},
{"\x0D", ErrorBadDecryptControlChar},
{"\x0E", ErrorBadDecryptControlChar},
{"\x0F", ErrorBadDecryptControlChar},
{"\x10", ErrorBadDecryptControlChar},
{"\x11", ErrorBadDecryptControlChar},
{"\x12", ErrorBadDecryptControlChar},
{"\x13", ErrorBadDecryptControlChar},
{"\x14", ErrorBadDecryptControlChar},
{"\x15", ErrorBadDecryptControlChar},
{"\x16", ErrorBadDecryptControlChar},
{"\x17", ErrorBadDecryptControlChar},
{"\x18", ErrorBadDecryptControlChar},
{"\x19", ErrorBadDecryptControlChar},
{"\x1A", ErrorBadDecryptControlChar},
{"\x1B", ErrorBadDecryptControlChar},
{"\x1C", ErrorBadDecryptControlChar},
{"\x1D", ErrorBadDecryptControlChar},
{"\x1E", ErrorBadDecryptControlChar},
{"\x1F", ErrorBadDecryptControlChar},
{"\x20", nil},
{"\x7E", nil},
{"\x7F", ErrorBadDecryptControlChar},
{"£100", nil},
{`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil},
{"£100", nil},
// Following tests from http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
{"a", nil}, // Valid ASCII
{"\xc3\xb1", nil}, // Valid 2 Octet Sequence
{"\xc3\x28", ErrorBadDecryptUTF8}, // Invalid 2 Octet Sequence
{"\xa0\xa1", ErrorBadDecryptUTF8}, // Invalid Sequence Identifier
{"\xe2\x82\xa1", nil}, // Valid 3 Octet Sequence
{"\xe2\x28\xa1", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 2nd Octet)
{"\xe2\x82\x28", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 3rd Octet)
{"\xf0\x90\x8c\xbc", nil}, // Valid 4 Octet Sequence
{"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 2nd Octet)
{"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 3rd Octet)
{"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 4th Octet)
{"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 5 Octet Sequence (but not Unicode!)
{"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!)
} {
actual := checkValidString([]byte(test.in))
assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
}
}
func TestEncodeFileName(t *testing.T) {
for _, test := range []struct {
in string
expected string
}{
{"", ""},
{"1", "64"},
{"12", "64p0"},
{"123", "64p36"},
{"1234", "64p36d0"},
{"12345", "64p36d1l"},
{"123456", "64p36d1l6o"},
{"1234567", "64p36d1l6org"},
{"12345678", "64p36d1l6orjg"},
{"123456789", "64p36d1l6orjge8"},
{"1234567890", "64p36d1l6orjge9g"},
{"12345678901", "64p36d1l6orjge9g64"},
{"123456789012", "64p36d1l6orjge9g64p0"},
{"1234567890123", "64p36d1l6orjge9g64p36"},
{"12345678901234", "64p36d1l6orjge9g64p36d0"},
{"123456789012345", "64p36d1l6orjge9g64p36d1l"},
{"1234567890123456", "64p36d1l6orjge9g64p36d1l6o"},
} {
actual := encodeFileName([]byte(test.in))
assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
recovered, err := decodeFileName(test.expected)
assert.NoError(t, err)
assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected))
in := strings.ToUpper(test.expected)
recovered, err = decodeFileName(in)
assert.NoError(t, err)
assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in))
}
}
func TestDecodeFileName(t *testing.T) {
// We've tested decoding the valid ones above, now concentrate on the invalid ones
for _, test := range []struct {
in string
expectedErr error
}{
{"64=", ErrorBadBase32Encoding},
{"!", base32.CorruptInputError(0)},
{"hello=hello", base32.CorruptInputError(5)},
} {
actual, actualErr := decodeFileName(test.in)
assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
}
}
func TestEncryptSegment(t *testing.T) {
c, _ := newCipher(0, "")
for _, test := range []struct {
in string
expected string
}{
{"", ""},
{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
{"12", "l42g6771hnv3an9cgc8cr2n1ng"},
{"123", "qgm4avr35m5loi1th53ato71v0"},
{"1234", "8ivr2e9plj3c3esisjpdisikos"},
{"12345", "rh9vu63q3o29eqmj4bg6gg7s44"},
{"123456", "bn717l3alepn75b2fb2ejmi4b4"},
{"1234567", "n6bo9jmb1qe3b1ogtj5qkf19k8"},
{"12345678", "u9t24j7uaq94dh5q53m3s4t9ok"},
{"123456789", "37hn305g6j12d1g0kkrl7ekbs4"},
{"1234567890", "ot8d91eplaglb62k2b1trm2qv0"},
{"12345678901", "h168vvrgb53qnrtvvmb378qrcs"},
{"123456789012", "s3hsdf9e29ithrqbjqu01t8q2s"},
{"1234567890123", "cf3jimlv1q2oc553mv7s3mh3eo"},
{"12345678901234", "moq0uqdlqrblrc5pa5u5c7hq9g"},
{"123456789012345", "eeam3li4rnommi3a762h5n7meg"},
{"1234567890123456", "mijbj0frqf6ms7frcr6bd9h0env53jv96pjaaoirk7forcgpt70g"},
} {
actual := c.encryptSegment(test.in)
assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in))
recovered, err := c.decryptSegment(test.expected)
assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
in := strings.ToUpper(test.expected)
recovered, err = c.decryptSegment(in)
assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in))
assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in))
}
}
func TestDecryptSegment(t *testing.T) {
// We've tested the forwards above, now concentrate on the errors
c, _ := newCipher(0, "")
for _, test := range []struct {
in string
expectedErr error
}{
{"64=", ErrorBadBase32Encoding},
{"!", base32.CorruptInputError(0)},
{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
{c.encryptSegment("\x01"), ErrorBadDecryptControlChar},
{c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8},
} {
actual, actualErr := c.decryptSegment(test.in)
assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
}
}
func TestSpreadName(t *testing.T) {
for _, test := range []struct {
n int
in string
expected string
}{
{3, "", ""},
{0, "abcdefg", "abcdefg"},
{1, "abcdefg", "a/abcdefg"},
{2, "abcdefg", "a/b/abcdefg"},
{3, "abcdefg", "a/b/c/abcdefg"},
{4, "abcdefg", "a/b/c/d/abcdefg"},
{4, "abcd", "a/b/c/d/abcd"},
{4, "abc", "a/b/c/abc"},
{4, "ab", "a/b/ab"},
{4, "a", "a/a"},
} {
actual := spreadName(test.n, test.in)
assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d,%q", test.n, test.in))
recovered, err := unspreadName(test.expected)
assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
}
}
func TestUnspreadName(t *testing.T) {
// We've tested the forwards above, now concentrate on the errors
for _, test := range []struct {
in string
expectedErr error
}{
{"aa/bc", ErrorBadSpreadNotSingleChar},
{"/", ErrorBadSpreadNotSingleChar},
{"a/", ErrorBadSpreadResultTooShort},
{"a/b/c/ab", ErrorBadSpreadResultTooShort},
{"a/b/x/abc", ErrorBadSpreadDidntMatch},
{"a/b/c/ABC", nil},
} {
actual, actualErr := unspreadName(test.in)
assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
}
}
func TestEncryptName(t *testing.T) {
// First no flatten
c, _ := newCipher(0, "")
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptName("1"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptName("1/12"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptName("1/12/123"))
// Now with flatten
c, _ = newCipher(3, "")
assert.Equal(t, "k/g/t/kgtickdcigo7600huebjl3ubu4", c.EncryptName("1/12/123"))
}
func TestDecryptName(t *testing.T) {
for _, test := range []struct {
flatten int
in string
expected string
expectedErr error
}{
{0, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
{0, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
{3, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
{1, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
{1, "k/g/t/i/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
{1, "k/x/t/i/kgtickdcigo7600huebjl3ubu4", "", ErrorBadSpreadDidntMatch},
} {
c, _ := newCipher(test.flatten, "")
actual, actualErr := c.DecryptName(test.in)
what := fmt.Sprintf("Testing %q (flatten=%d)", test.in, test.flatten)
assert.Equal(t, test.expected, actual, what)
assert.Equal(t, test.expectedErr, actualErr, what)
}
}
func TestEncryptedSize(t *testing.T) {
c, _ := newCipher(0, "")
for _, test := range []struct {
in int64
expected int64
}{
{0, 32},
{1, 32 + 16 + 1},
{65536, 32 + 16 + 65536},
{65537, 32 + 16 + 65536 + 16 + 1},
{1 << 20, 32 + 16*(16+65536)},
{(1 << 20) + 65535, 32 + 16*(16+65536) + 16 + 65535},
{1 << 30, 32 + 16384*(16+65536)},
{(1 << 40) + 1, 32 + 16777216*(16+65536) + 16 + 1},
} {
actual := c.EncryptedSize(test.in)
assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d", test.in))
recovered, err := c.DecryptedSize(test.expected)
assert.NoError(t, err, fmt.Sprintf("Testing reverse %d", test.expected))
assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %d", test.expected))
}
}
func TestDecryptedSize(t *testing.T) {
// Test the errors since we tested the reverse above
c, _ := newCipher(0, "")
for _, test := range []struct {
in int64
expectedErr error
}{
{0, ErrorEncryptedFileTooShort},
{0, ErrorEncryptedFileTooShort},
{1, ErrorEncryptedFileTooShort},
{7, ErrorEncryptedFileTooShort},
{32 + 1, ErrorEncryptedFileBadHeader},
{32 + 16, ErrorEncryptedFileBadHeader},
{32 + 16 + 65536 + 1, ErrorEncryptedFileBadHeader},
{32 + 16 + 65536 + 16, ErrorEncryptedFileBadHeader},
} {
_, actualErr := c.DecryptedSize(test.in)
assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("Testing %d", test.in))
}
}
func TestNoncePointer(t *testing.T) {
var x nonce
assert.Equal(t, (*[24]byte)(&x), x.pointer())
}
func TestNonceFromReader(t *testing.T) {
var x nonce
buf := bytes.NewBufferString("123456789abcdefghijklmno")
err := x.fromReader(buf)
assert.NoError(t, err)
assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
buf = bytes.NewBufferString("123456789abcdefghijklmn")
err = x.fromReader(buf)
assert.Error(t, err, "short read of nonce")
}
func TestNonceFromBuf(t *testing.T) {
var x nonce
buf := []byte("123456789abcdefghijklmnoXXXXXXXX")
x.fromBuf(buf)
assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x)
buf = []byte("0123456789abcdefghijklmn")
x.fromBuf(buf)
assert.Equal(t, nonce{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n'}, x)
buf = []byte("0123456789abcdefghijklm")
assert.Panics(t, func() { x.fromBuf(buf) })
}
func TestNonceIncrement(t *testing.T) {
for _, test := range []struct {
in nonce
out nonce
}{
{
nonce{0x00},
nonce{0x01},
},
{
nonce{0xFF},
nonce{0x00, 0x01},
},
{
nonce{0xFF, 0xFF},
nonce{0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
},
} {
x := test.in
x.increment()
assert.Equal(t, test.out, x)
}
}
// randomSource can read or write a random sequence
type randomSource struct {
counter int64
size int64
}
func newRandomSource(size int64) *randomSource {
return &randomSource{
size: size,
}
}
func (r *randomSource) next() byte {
r.counter++
return byte(r.counter % 257)
}
func (r *randomSource) Read(p []byte) (n int, err error) {
for i := range p {
if r.counter >= r.size {
err = io.EOF
break
}
p[i] = r.next()
n++
}
return n, err
}
func (r *randomSource) Write(p []byte) (n int, err error) {
for i := range p {
if p[i] != r.next() {
return 0, errors.Errorf("Error in stream at %d", r.counter)
}
}
return len(p), nil
}
func (r *randomSource) Close() error { return nil }
// Check interfaces
var (
_ io.ReadCloser = (*randomSource)(nil)
_ io.WriteCloser = (*randomSource)(nil)
)
// Test test infrastructure first!
func TestRandomSource(t *testing.T) {
source := newRandomSource(1E8)
sink := newRandomSource(1E8)
n, err := io.Copy(sink, source)
assert.NoError(t, err)
assert.Equal(t, int64(1E8), n)
source = newRandomSource(1E8)
buf := make([]byte, 16)
_, _ = source.Read(buf)
sink = newRandomSource(1E8)
n, err = io.Copy(sink, source)
assert.Error(t, err, "Error in stream")
}
type zeroes struct{}
func (z *zeroes) Read(p []byte) (n int, err error) {
for i := range p {
p[i] = 0
n++
}
return n, nil
}
// Test encrypt decrypt with different buffer sizes
func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
c, err := newCipher(0, "")
assert.NoError(t, err)
c.cryptoRand = &zeroes{} // zero out the nonce
buf := make([]byte, bufSize)
source := newRandomSource(copySize)
encrypted, err := c.newEncrypter(source)
assert.NoError(t, err)
decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
assert.NoError(t, err)
sink := newRandomSource(copySize)
n, err := io.CopyBuffer(sink, decrypted, buf)
assert.NoError(t, err)
assert.Equal(t, copySize, n)
blocks := copySize / blockSize
if (copySize % blockSize) != 0 {
blocks++
}
var expectedNonce = nonce{byte(blocks), byte(blocks >> 8), byte(blocks >> 16), byte(blocks >> 32)}
assert.Equal(t, expectedNonce, encrypted.nonce)
assert.Equal(t, expectedNonce, decrypted.nonce)
}
func TestEncryptDecrypt1(t *testing.T) {
testEncryptDecrypt(t, 1, 1E7)
}
func TestEncryptDecrypt32(t *testing.T) {
testEncryptDecrypt(t, 32, 1E8)
}
func TestEncryptDecrypt4096(t *testing.T) {
testEncryptDecrypt(t, 4096, 1E8)
}
func TestEncryptDecrypt65536(t *testing.T) {
testEncryptDecrypt(t, 65536, 1E8)
}
func TestEncryptDecrypt65537(t *testing.T) {
testEncryptDecrypt(t, 65537, 1E8)
}
var (
file0 = []byte{
0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
}
file1 = []byte{
0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
0x09, 0x5b, 0x44, 0x6c, 0xd6, 0x23, 0x7b, 0xbc, 0xb0, 0x8d, 0x09, 0xfb, 0x52, 0x4c, 0xe5, 0x65,
0xAA,
}
file16 = []byte{
0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
0xb9, 0xc4, 0x55, 0x2a, 0x27, 0x10, 0x06, 0x29, 0x18, 0x96, 0x0a, 0x3e, 0x60, 0x8c, 0x29, 0xb9,
0xaa, 0x8a, 0x5e, 0x1e, 0x16, 0x5b, 0x6d, 0x07, 0x5d, 0xe4, 0xe9, 0xbb, 0x36, 0x7f, 0xd6, 0xd4,
}
)
func TestEncryptData(t *testing.T) {
for _, test := range []struct {
in []byte
expected []byte
}{
{[]byte{}, file0},
{[]byte{1}, file1},
{[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16},
} {
c, err := newCipher(0, "")
assert.NoError(t, err)
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
// Check encode works
buf := bytes.NewBuffer(test.in)
encrypted, err := c.EncryptData(buf)
assert.NoError(t, err)
out, err := ioutil.ReadAll(encrypted)
assert.NoError(t, err)
assert.Equal(t, test.expected, out)
// Check we can decode the data properly too...
buf = bytes.NewBuffer(out)
decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
assert.NoError(t, err)
out, err = ioutil.ReadAll(decrypted)
assert.NoError(t, err)
assert.Equal(t, test.in, out)
}
}
func TestNewEncrypter(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
z := &zeroes{}
fh, err := c.newEncrypter(z)
assert.NoError(t, err)
assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce)
assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32])
// Test error path
c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn")
fh, err = c.newEncrypter(z)
assert.Nil(t, fh)
assert.Error(t, err, "short read of nonce")
}
type errorReader struct {
err error
}
func (er errorReader) Read(p []byte) (n int, err error) {
return 0, er.err
}
type closeDetector struct {
io.Reader
closed int
}
func newCloseDetector(in io.Reader) *closeDetector {
return &closeDetector{
Reader: in,
}
}
func (c *closeDetector) Close() error {
c.closed++
return nil
}
func TestNewDecrypter(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
cd := newCloseDetector(bytes.NewBuffer(file0))
fh, err := c.newDecrypter(cd)
assert.NoError(t, err)
// check nonce is in place
assert.Equal(t, file0[8:32], fh.nonce[:])
assert.Equal(t, 0, cd.closed)
// Test error paths
for i := range file0 {
cd := newCloseDetector(bytes.NewBuffer(file0[:i]))
fh, err = c.newDecrypter(cd)
assert.Nil(t, fh)
assert.Error(t, err, ErrorEncryptedFileTooShort.Error())
assert.Equal(t, 1, cd.closed)
}
er := &errorReader{errors.New("potato")}
cd = newCloseDetector(er)
fh, err = c.newDecrypter(cd)
assert.Nil(t, fh)
assert.Error(t, err, "potato")
assert.Equal(t, 1, cd.closed)
// bad magic
file0copy := make([]byte, len(file0))
copy(file0copy, file0)
for i := range fileMagic {
file0copy[i] ^= 0x1
cd := newCloseDetector(bytes.NewBuffer(file0copy))
fh, err := c.newDecrypter(cd)
assert.Nil(t, fh)
assert.Error(t, err, ErrorEncryptedBadMagic.Error())
file0copy[i] ^= 0x1
assert.Equal(t, 1, cd.closed)
}
}
func TestDecrypterRead(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
// Test truncating the header
for i := 1; i < blockHeaderSize; i++ {
cd := newCloseDetector(bytes.NewBuffer(file1[:len(file1)-i]))
fh, err := c.newDecrypter(cd)
assert.NoError(t, err)
_, err = ioutil.ReadAll(fh)
assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
assert.Equal(t, 0, cd.closed)
}
// Test producing an error on the file on Read the underlying file
in1 := bytes.NewBuffer(file1)
in2 := &errorReader{errors.New("potato")}
in := io.MultiReader(in1, in2)
cd := newCloseDetector(in)
fh, err := c.newDecrypter(cd)
assert.NoError(t, err)
_, err = ioutil.ReadAll(fh)
assert.Error(t, err, "potato")
assert.Equal(t, 0, cd.closed)
// Test corrupting the input
// shouldn't be able to corrupt any byte without some sort of error
file16copy := make([]byte, len(file16))
copy(file16copy, file16)
for i := range file16copy {
file16copy[i] ^= 0xFF
fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
if i < fileMagicSize {
assert.Error(t, err, ErrorEncryptedBadMagic.Error())
assert.Nil(t, fh)
} else {
assert.NoError(t, err)
_, err = ioutil.ReadAll(fh)
assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
}
file16copy[i] ^= 0xFF
}
}
func TestDecrypterClose(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
cd := newCloseDetector(bytes.NewBuffer(file16))
fh, err := c.newDecrypter(cd)
assert.NoError(t, err)
assert.Equal(t, 0, cd.closed)
// close before reading
assert.Equal(t, nil, fh.err)
err = fh.Close()
assert.Equal(t, ErrorFileClosed, fh.err)
assert.Equal(t, 1, cd.closed)
// double close
err = fh.Close()
assert.Error(t, err, ErrorFileClosed.Error())
assert.Equal(t, 1, cd.closed)
// try again reading the file this time
cd = newCloseDetector(bytes.NewBuffer(file1))
fh, err = c.newDecrypter(cd)
assert.NoError(t, err)
assert.Equal(t, 0, cd.closed)
// close after reading
out, err := ioutil.ReadAll(fh)
assert.NoError(t, err)
assert.Equal(t, []byte{1}, out)
assert.Equal(t, io.EOF, fh.err)
err = fh.Close()
assert.Equal(t, ErrorFileClosed, fh.err)
assert.Equal(t, 1, cd.closed)
}
func TestPutGetBlock(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
block := c.getBlock()
c.putBlock(block)
c.putBlock(block)
assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) })
}
func TestKey(t *testing.T) {
c, err := newCipher(0, "")
assert.NoError(t, err)
// Check zero keys OK
assert.Equal(t, [32]byte{}, c.dataKey)
assert.Equal(t, [32]byte{}, c.nameKey)
assert.Equal(t, [16]byte{}, c.nameTweak)
require.NoError(t, c.Key("potato"))
assert.Equal(t, [32]byte{0x74, 0x55, 0xC7, 0x1A, 0xB1, 0x7C, 0x86, 0x5B, 0x84, 0x71, 0xF4, 0x7B, 0x79, 0xAC, 0xB0, 0x7E, 0xB3, 0x1D, 0x56, 0x78, 0xB8, 0x0C, 0x7E, 0x2E, 0xAF, 0x4F, 0xC8, 0x06, 0x6A, 0x9E, 0xE4, 0x68}, c.dataKey)
assert.Equal(t, [32]byte{0x76, 0x5D, 0xA2, 0x7A, 0xB1, 0x5D, 0x77, 0xF9, 0x57, 0x96, 0x71, 0x1F, 0x7B, 0x93, 0xAD, 0x63, 0xBB, 0xB4, 0x84, 0x07, 0x2E, 0x71, 0x80, 0xA8, 0xD1, 0x7A, 0x9B, 0xBE, 0xC1, 0x42, 0x70, 0xD0}, c.nameKey)
assert.Equal(t, [16]byte{0xC1, 0x8D, 0x59, 0x32, 0xF5, 0x5B, 0x28, 0x28, 0xC5, 0xE1, 0xE8, 0x72, 0x15, 0x52, 0x03, 0x10}, c.nameTweak)
require.NoError(t, c.Key("Potato"))
assert.Equal(t, [32]byte{0xAE, 0xEA, 0x6A, 0xD3, 0x47, 0xDF, 0x75, 0xB9, 0x63, 0xCE, 0x12, 0xF5, 0x76, 0x23, 0xE9, 0x46, 0xD4, 0x2E, 0xD8, 0xBF, 0x3E, 0x92, 0x8B, 0x39, 0x24, 0x37, 0x94, 0x13, 0x3E, 0x5E, 0xF7, 0x5E}, c.dataKey)
assert.Equal(t, [32]byte{0x54, 0xF7, 0x02, 0x6E, 0x8A, 0xFC, 0x56, 0x0A, 0x86, 0x63, 0x6A, 0xAB, 0x2C, 0x9C, 0x51, 0x62, 0xE5, 0x1A, 0x12, 0x23, 0x51, 0x83, 0x6E, 0xAF, 0x50, 0x42, 0x0F, 0x98, 0x1C, 0x86, 0x0A, 0x19}, c.nameKey)
assert.Equal(t, [16]byte{0xF8, 0xC1, 0xB6, 0x27, 0x2D, 0x52, 0x9B, 0x4A, 0x8F, 0xDA, 0xEB, 0x42, 0x4A, 0x28, 0xDD, 0xF3}, c.nameTweak)
require.NoError(t, c.Key(""))
assert.Equal(t, [32]byte{}, c.dataKey)
assert.Equal(t, [32]byte{}, c.nameKey)
assert.Equal(t, [16]byte{}, c.nameTweak)
}

445
crypt/crypt.go Normal file
View file

@ -0,0 +1,445 @@
// Package crypt provides wrappers for Fs and Object which implement encryption
package crypt
import (
"fmt"
"io"
"path"
"strings"
"sync"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
Name: "crypt",
Description: "Encrypt/Decrypt a remote",
NewFs: NewFs,
Options: []fs.Option{{
Name: "remote",
Help: "Remote to encrypt/decrypt.",
}, {
Name: "flatten",
Help: "Flatten the directory structure - more secure, less useful - see docs for tradeoffs.",
Examples: []fs.OptionExample{
{
Value: "0",
Help: "Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.",
}, {
Value: "1",
Help: "Spread files over 1 directory good for <10,000 files.",
}, {
Value: "2",
Help: "Spread files over 32 directories good for <320,000 files.",
}, {
Value: "3",
Help: "Spread files over 1024 directories good for <10,000,000 files.",
}, {
Value: "4",
Help: "Spread files over 32,768 directories good for <320,000,000 files.",
}, {
Value: "5",
Help: "Spread files over 1,048,576 levels good for <10,000,000,000 files.",
},
},
}, {
Name: "password",
Help: "Password or pass phrase for encryption.",
IsPassword: true,
}},
})
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, rpath string) (fs.Fs, error) {
flatten := fs.ConfigFile.MustInt(name, "flatten", 0)
password := fs.ConfigFile.MustValue(name, "password", "")
if password == "" {
return nil, errors.New("password not set in config file")
}
password, err := fs.Reveal(password)
if err != nil {
return nil, errors.Wrap(err, "failed to decrypt password")
}
cipher, err := newCipher(flatten, password)
if err != nil {
return nil, errors.Wrap(err, "failed to make cipher")
}
remote := fs.ConfigFile.MustValue(name, "remote")
remotePath := path.Join(remote, cipher.EncryptName(rpath))
wrappedFs, err := fs.NewFs(remotePath)
if err != fs.ErrorIsFile && err != nil {
return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
}
f := &Fs{
Fs: wrappedFs,
cipher: cipher,
flatten: flatten,
}
return f, err
}
// Fs represents a wrapped fs.Fs
type Fs struct {
fs.Fs
cipher Cipher
flatten int
}
// String returns a description of the FS
func (f *Fs) String() string {
return fmt.Sprintf("%s with cipher", f.Fs.String())
}
// List the Fs into a channel
func (f *Fs) List(opts fs.ListOpts, dir string) {
f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptName(dir))
}
// NewObject finds the Object at remote.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
o, err := f.Fs.NewObject(f.cipher.EncryptName(remote))
if err != nil {
return nil, err
}
return f.newObject(o), nil
}
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
wrappedIn, err := f.cipher.EncryptData(in)
if err != nil {
return nil, err
}
o, err := f.Fs.Put(wrappedIn, f.newObjectInfo(src))
if err != nil {
return nil, err
}
return f.newObject(o), nil
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() fs.HashSet {
return fs.HashSet(fs.HashNone)
}
// Purge all files in the root and the root directory
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
func (f *Fs) Purge() error {
do, ok := f.Fs.(fs.Purger)
if !ok {
return fs.ErrorCantPurge
}
return do.Purge()
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
do, ok := f.Fs.(fs.Copier)
if !ok {
return nil, fs.ErrorCantCopy
}
o, ok := src.(*Object)
if !ok {
return nil, fs.ErrorCantCopy
}
oResult, err := do.Copy(o.Object, f.cipher.EncryptName(remote))
if err != nil {
return nil, err
}
return f.newObject(oResult), nil
}
// Move src to this remote using server side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
do, ok := f.Fs.(fs.Mover)
if !ok {
return nil, fs.ErrorCantMove
}
o, ok := src.(*Object)
if !ok {
return nil, fs.ErrorCantCopy
}
oResult, err := do.Move(o.Object, f.cipher.EncryptName(remote))
if err != nil {
return nil, err
}
return f.newObject(oResult), nil
}
// UnWrap returns the Fs that this Fs is wrapping
func (f *Fs) UnWrap() fs.Fs {
return f.Fs
}
// Object describes a wrapped for being read from the Fs
//
// This decrypts the remote name and decrypts the data
type Object struct {
fs.Object
f *Fs
}
func (f *Fs) newObject(o fs.Object) *Object {
return &Object{
Object: o,
f: f,
}
}
// Fs returns read only access to the Fs that this object is part of
func (o *Object) Fs() fs.Info {
return o.f
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.Remote()
}
// Remote returns the remote path
func (o *Object) Remote() string {
remote := o.Object.Remote()
decryptedName, err := o.f.cipher.DecryptName(remote)
if err != nil {
fs.Debug(remote, "Undecryptable file name: %v", err)
return remote
}
return decryptedName
}
// Size returns the size of the file
func (o *Object) Size() int64 {
size, err := o.f.cipher.DecryptedSize(o.Object.Size())
if err != nil {
fs.Debug(o, "Bad size for decrypt: %v", err)
}
return size
}
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *Object) Hash(hash fs.HashType) (string, error) {
return "", nil
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open() (io.ReadCloser, error) {
in, err := o.Object.Open()
if err != nil {
return in, err
}
return o.f.cipher.DecryptData(in)
}
// Update in to the object with the modTime given of the given size
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
wrappedIn, err := o.f.cipher.EncryptData(in)
if err != nil {
return err
}
return o.Object.Update(wrappedIn, o.f.newObjectInfo(src))
}
// newDir returns a dir with the Name decrypted
func (f *Fs) newDir(dir *fs.Dir) *fs.Dir {
new := *dir
remote := dir.Name
decryptedRemote, err := f.cipher.DecryptName(remote)
if err != nil {
fs.Debug(remote, "Undecryptable dir name: %v", err)
} else {
new.Name = decryptedRemote
}
return &new
}
// ObjectInfo describes a wrapped fs.ObjectInfo for being the source
//
// This encrypts the remote name and adjusts the size
type ObjectInfo struct {
fs.ObjectInfo
f *Fs
}
func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo {
return &ObjectInfo{
ObjectInfo: src,
f: f,
}
}
// Fs returns read only access to the Fs that this object is part of
func (o *ObjectInfo) Fs() fs.Info {
return o.f
}
// Remote returns the remote path
func (o *ObjectInfo) Remote() string {
return o.f.cipher.EncryptName(o.ObjectInfo.Remote())
}
// Size returns the size of the file
func (o *ObjectInfo) Size() int64 {
return o.f.cipher.EncryptedSize(o.ObjectInfo.Size())
}
// ListOpts wraps a listopts decrypting the directory listing and
// replacing the Objects
type ListOpts struct {
fs.ListOpts
f *Fs
dir string // dir we are listing
mu sync.Mutex // to protect dirs
dirs map[string]struct{} // keep track of synthetic directory objects added
}
// Make a ListOpts wrapper
func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts {
if dir != "" {
dir += "/"
}
return &ListOpts{
ListOpts: lo,
f: f,
dir: dir,
dirs: make(map[string]struct{}),
}
}
// Level gets the recursion level for this listing.
//
// Fses may ignore this, but should implement it for improved efficiency if possible.
//
// Level 1 means list just the contents of the directory
//
// Each returned item must have less than level `/`s in.
func (lo *ListOpts) Level() int {
// If flattened recurse fully
if lo.f.flatten > 0 {
return fs.MaxLevel
}
return lo.ListOpts.Level()
}
// addSyntheticDirs makes up directory objects for the path passed in
func (lo *ListOpts) addSyntheticDirs(path string) {
lo.mu.Lock()
defer lo.mu.Unlock()
for {
i := strings.LastIndexByte(path, '/')
if i < 0 {
break
}
path = path[:i]
if path == "" {
break
}
if _, found := lo.dirs[path]; found {
break
}
slashes := strings.Count(path, "/")
if slashes < lo.ListOpts.Level() {
lo.ListOpts.AddDir(&fs.Dir{Name: path})
}
lo.dirs[path] = struct{}{}
}
}
// Add an object to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
remote := obj.Remote()
decryptedRemote, err := lo.f.cipher.DecryptName(remote)
if err != nil {
fs.Debug(remote, "Skipping undecryptable file name: %v", err)
return lo.ListOpts.IsFinished()
}
// If flattened add synthetic directories
if lo.f.flatten > 0 {
lo.addSyntheticDirs(decryptedRemote)
slashes := strings.Count(decryptedRemote, "/")
if slashes >= lo.ListOpts.Level() {
return lo.ListOpts.IsFinished()
}
}
return lo.ListOpts.Add(lo.f.newObject(obj))
}
// AddDir adds a directory to the output.
// If the function returns true, the operation has been aborted.
// Multiple goroutines can safely add objects concurrently.
func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
// If flattened we don't add any directories from the underlying remote
if lo.f.flatten > 0 {
return lo.ListOpts.IsFinished()
}
remote := dir.Name
_, err := lo.f.cipher.DecryptName(remote)
if err != nil {
fs.Debug(remote, "Skipping undecryptable dir name: %v", err)
return lo.ListOpts.IsFinished()
}
return lo.ListOpts.AddDir(lo.f.newDir(dir))
}
// IncludeDirectory returns whether this directory should be
// included in the listing (and recursed into or not).
func (lo *ListOpts) IncludeDirectory(remote string) bool {
// If flattened we look in all directories
if lo.f.flatten > 0 {
return true
}
decryptedRemote, err := lo.f.cipher.DecryptName(remote)
if err != nil {
fs.Debug(remote, "Not including undecryptable directory name: %v", err)
return false
}
return lo.ListOpts.IncludeDirectory(decryptedRemote)
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
// _ fs.DirMover = (*Fs)(nil)
// _ fs.PutUncheckeder = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.ObjectInfo = (*ObjectInfo)(nil)
_ fs.Object = (*Object)(nil)
_ fs.ListOpts = (*ListOpts)(nil)
)

59
crypt/crypt_test.go Normal file
View file

@ -0,0 +1,59 @@
// Test Crypt filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: make gen_tests
package crypt_test
import (
"testing"
"github.com/ncw/rclone/crypt"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
_ "github.com/ncw/rclone/local"
)
func init() {
fstests.NilObject = fs.Object((*crypt.Object)(nil))
fstests.RemoteName = "TestCrypt:"
}
// Generic tests for the Fs
func TestInit(t *testing.T) { fstests.TestInit(t) }
func TestFsString(t *testing.T) { fstests.TestFsString(t) }
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }

63
crypt/pkcs7/pkcs7.go Normal file
View file

@ -0,0 +1,63 @@
// Package pkcs7 implements PKCS#7 padding
//
// This is a standard way of encoding variable length buffers into
// buffers which are a multiple of an underlying crypto block size.
package pkcs7
import "github.com/pkg/errors"
// Errors Unpad can return
var (
ErrorPaddingNotFound = errors.New("Bad PKCS#7 padding - not padded")
ErrorPaddingNotAMultiple = errors.New("Bad PKCS#7 padding - not a multiple of blocksize")
ErrorPaddingTooLong = errors.New("Bad PKCS#7 padding - too long")
ErrorPaddingTooShort = errors.New("Bad PKCS#7 padding - too short")
ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same")
)
// Pad buf using PKCS#7 to a multiple of n.
//
// Appends the padding to buf - make a copy of it first if you don't
// want it modified.
func Pad(n int, buf []byte) []byte {
if n <= 1 || n >= 256 {
panic("bad multiple")
}
length := len(buf)
padding := n - (length % n)
for i := 0; i < padding; i++ {
buf = append(buf, byte(padding))
}
if (len(buf) % n) != 0 {
panic("padding failed")
}
return buf
}
// Unpad buf using PKCS#7 from a multiple of n returning a slice of
// buf or an error if malformed.
func Unpad(n int, buf []byte) ([]byte, error) {
if n <= 1 || n >= 256 {
panic("bad multiple")
}
length := len(buf)
if length == 0 {
return nil, ErrorPaddingNotFound
}
if (length % n) != 0 {
return nil, ErrorPaddingNotAMultiple
}
padding := int(buf[length-1])
if padding > n {
return nil, ErrorPaddingTooLong
}
if padding == 0 {
return nil, ErrorPaddingTooShort
}
for i := 0; i < padding; i++ {
if buf[length-1-i] != byte(padding) {
return nil, ErrorPaddingNotAllTheSame
}
}
return buf[:length-padding], nil
}

73
crypt/pkcs7/pkcs7_test.go Normal file
View file

@ -0,0 +1,73 @@
package pkcs7
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPad(t *testing.T) {
for _, test := range []struct {
n int
in string
expected string
}{
{8, "", "\x08\x08\x08\x08\x08\x08\x08\x08"},
{8, "1", "1\x07\x07\x07\x07\x07\x07\x07"},
{8, "12", "12\x06\x06\x06\x06\x06\x06"},
{8, "123", "123\x05\x05\x05\x05\x05"},
{8, "1234", "1234\x04\x04\x04\x04"},
{8, "12345", "12345\x03\x03\x03"},
{8, "123456", "123456\x02\x02"},
{8, "1234567", "1234567\x01"},
{8, "abcdefgh", "abcdefgh\x08\x08\x08\x08\x08\x08\x08\x08"},
{8, "abcdefgh1", "abcdefgh1\x07\x07\x07\x07\x07\x07\x07"},
{8, "abcdefgh12", "abcdefgh12\x06\x06\x06\x06\x06\x06"},
{8, "abcdefgh123", "abcdefgh123\x05\x05\x05\x05\x05"},
{8, "abcdefgh1234", "abcdefgh1234\x04\x04\x04\x04"},
{8, "abcdefgh12345", "abcdefgh12345\x03\x03\x03"},
{8, "abcdefgh123456", "abcdefgh123456\x02\x02"},
{8, "abcdefgh1234567", "abcdefgh1234567\x01"},
{8, "abcdefgh12345678", "abcdefgh12345678\x08\x08\x08\x08\x08\x08\x08\x08"},
{16, "", "\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10"},
{16, "a", "a\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f"},
} {
actual := Pad(test.n, []byte(test.in))
assert.Equal(t, test.expected, string(actual), fmt.Sprintf("Pad %d %q", test.n, test.in))
recovered, err := Unpad(test.n, actual)
assert.NoError(t, err)
assert.Equal(t, []byte(test.in), recovered, fmt.Sprintf("Unpad %d %q", test.n, test.in))
}
assert.Panics(t, func() { Pad(1, []byte("")) }, "bad multiple")
assert.Panics(t, func() { Pad(256, []byte("")) }, "bad multiple")
}
func TestUnpad(t *testing.T) {
// We've tested the OK decoding in TestPad, now test the error cases
for _, test := range []struct {
n int
in string
err error
}{
{8, "", ErrorPaddingNotFound},
{8, "1", ErrorPaddingNotAMultiple},
{8, "12", ErrorPaddingNotAMultiple},
{8, "123", ErrorPaddingNotAMultiple},
{8, "1234", ErrorPaddingNotAMultiple},
{8, "12345", ErrorPaddingNotAMultiple},
{8, "123456", ErrorPaddingNotAMultiple},
{8, "1234567", ErrorPaddingNotAMultiple},
{8, "1234567\xFF", ErrorPaddingTooLong},
{8, "1234567\x09", ErrorPaddingTooLong},
{8, "1234567\x00", ErrorPaddingTooShort},
{8, "123456\x01\x02", ErrorPaddingNotAllTheSame},
{8, "\x07\x08\x08\x08\x08\x08\x08\x08", ErrorPaddingNotAllTheSame},
} {
result, actualErr := Unpad(test.n, []byte(test.in))
assert.Equal(t, test.err, actualErr, fmt.Sprintf("Unpad %d %q", test.n, test.in))
assert.Equal(t, result, []byte(nil))
}
assert.Panics(t, func() { _, _ = Unpad(1, []byte("")) }, "bad multiple")
assert.Panics(t, func() { _, _ = Unpad(256, []byte("")) }, "bad multiple")
}

288
docs/content/crypt.md Normal file
View file

@ -0,0 +1,288 @@
---
title: "Crypt"
description: "Encryption overlay remote"
date: "2016-07-28"
---
<i class="fa fa-lock"></i>Crypt
----------------------------------------
The `crypt` remote encrypts and decrypts another remote.
To use it first set up the underlying remote following the config
instructions for that remote. You can also use a local pathname
instead of a remote which will encrypt and decrypt from that directory
which might be useful for encrypting onto a USB stick for example.
First check your chosen remote is working - we'll call it
`remote:path` in these docs. Note that anything inside `remote:path`
will be encrypted and anything outside won't. This means that if you
are using a bucket based remote (eg S3, B2, swift) then you should
probably put the bucket in the remote `s3:bucket`. If you just use
`s3:` then rclone will make encrypted bucket names too which may or
may not be what you want.
Now configure `crypt` using `rclone config`. We will call this one
`secret` to differentiate it from the `remote`.
```
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> secret
Type of storage to configure.
Choose a number from below, or type in your own value
1 / Amazon Drive
\ "amazon cloud drive"
2 / Amazon S3 (also Dreamhost, Ceph, Minio)
\ "s3"
3 / Backblaze B2
\ "b2"
4 / Dropbox
\ "dropbox"
5 / Encrypt/Decrypt a remote
\ "crypt"
6 / Google Cloud Storage (this is not Google Drive)
\ "google cloud storage"
7 / Google Drive
\ "drive"
8 / Hubic
\ "hubic"
9 / Local Disk
\ "local"
10 / Microsoft OneDrive
\ "onedrive"
11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
\ "swift"
12 / Yandex Disk
\ "yandex"
Storage> 5
Remote to encrypt/decrypt.
remote> remote:path
Flatten the directory structure - more secure, less useful - see docs for tradeoffs.
Choose a number from below, or type in your own value
1 / Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.
\ "0"
2 / Spread files over 1 directory good for <10,000 files.
\ "1"
3 / Spread files over 32 directories good for <320,000 files.
\ "2"
4 / Spread files over 1024 directories good for <10,000,000 files.
\ "3"
5 / Spread files over 32,768 directories good for <320,000,000 files.
\ "4"
6 / Spread files over 1,048,576 levels good for <10,000,000,000 files.
\ "5"
flatten> 1
Password or pass phrase for encryption.
Enter the password:
password:
Confirm the password:
password:
Remote config
--------------------
[secret]
remote = remote:path
flatten = 0
password = 0_gtCJ422bzwAWP0UN2lggrjhA-sSg
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
**Important** The password is stored in the config file is lightly
obscured so it isn't immediately obvious what it is. It is in no way
secure unless you use config file encryption.
A long passphrase is recommended, or you can use a random one. Note
that if you reconfigure rclone with the same password/passphrase
elsewhere it will be compatible - all the secrets used are derived
from that one password/passphrase.
Note that rclone does not encrypt
* file length - this can be calcuated within 16 bytes
* modification time - used for syncing
## Example ##
To test I made a little directory of files
```
plaintext/
├── file0.txt
├── file1.txt
└── subdir
├── file2.txt
├── file3.txt
└── subsubdir
└── file4.txt
```
Copy these to the remote and list them back
```
$ rclone -q copy plaintext secret:
$ rclone -q ls secret:
7 file1.txt
6 file0.txt
8 subdir/file2.txt
10 subdir/subsubdir/file4.txt
9 subdir/file3.txt
```
Now see what that looked like when encrypted
```
$ rclone -q ls remote:path
55 hagjclgavj2mbiqm6u6cnjjqcg
54 v05749mltvv1tf4onltun46gls
57 86vhrsv86mpbtd3a0akjuqslj8/dlj7fkq4kdq72emafg7a7s41uo
58 86vhrsv86mpbtd3a0akjuqslj8/7uu829995du6o42n32otfhjqp4/b9pausrfansjth5ob3jkdqd4lc
56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps
```
Note that this retains the directory structure which means you can do this
```
$ rclone -q ls secret:subdir
8 file2.txt
9 file3.txt
10 subsubdir/file4.txt
```
If you use the flattened flag then the listing will look and that last command will not work.
```
$ rclone -q ls remote:path
56 t/tsdtcpdu6g9dpamn6poqc248tll9dj5ok78a363etmq8ushr821g
57 g/gsrp2g0u85pgsi6kso74bjsrsafe11odpfln8qqpj6n9p20of0a0
55 h/hagjclgavj2mbiqm6u6cnjjqcg
58 4/4jsbao3dhi0jfoubt2oo493pbqmsshn92q01ddu7dg6428rlluhg
54 v/v05749mltvv1tf4onltun46gls
```
### Flattened vs non-Flattened ###
Pros and cons of each
Flattened
* hides directory structures
* identical file names won't have identical encrypted names
* can't use a sub path
* doesn't work: `rclone copy crypt:sub/dir /tmp/recovered`
* use: `rclone copy --include "/sub/dir/**" crypt: /tmp/recovered`
* will always have to recurse through the entire directory structure
* can't copy a single file directly
* doesn't work: `rclone copy crypt:path/to/file /tmp/recovered`
* use: `rclone copy --include "/path/to/file" crypt: /tmp/recovered`
Normal
* can use sub paths and copy single files
* directory structure visibile
* identical files names will have identical uploaded names
* can use shortcuts to shorten the directory recursion
You can swap between flattened levels without re-uploading your files.
## File formats ##
### File encryption ###
Files are encrypted 1:1 source file to destination object. The file
has a header and is divided into chunks.
#### Header ####
* 8 bytes magic string `RCLONE\x00\x00`
* 24 bytes Nonce (IV)
The initial nonce is generated from the operating systems crypto
strong random number genrator. The nonce is incremented for each
chunk read making sure each nonce is unique for each block written.
The chance of a nonce being re-used is miniscule. If you wrote an
exabyte of data (10¹⁸ bytes) you would have a probability of
approximately 2×10⁻³² of re-using a nonce.
#### Chunk ####
Each chunk will contain 64kB of data, except for the last one which
may have less data. The data chunk is in standard NACL secretbox
format. Secretbox uses XSalsa20 and Poly1305 to encrypt and
authenticate messages.
Each chunk contains:
* 16 Bytes of Poly1305 authenticator
* 1 - 65536 bytes XSalsa20 encrypted data
64k chunk size was chosen as the best performing chunk size (the
authenticator takes too much time below this and the performance drops
off due to cache effects above this). Note that these chunks are
buffered in memory so they can't be too big.
This uses a 32 byte (256 bit key) key derived from the user password.
#### Examples ####
1 byte file will encrypt to
* 32 bytes header
* 17 bytes data chunk
49 bytes total
1MB (1048576 bytes) file will encrypt to
* 32 bytes header
* 16 chunks of 65568 bytes
1049120 bytes total (a 0.05% overhead). This is the overhead for big
files.
### Name encryption ###
File names are encrypted by crypt. These are either encrypted segment
by segment - the path is broken up into `/` separated strings and
these are encrypted individually, or if working in flattened mode the
whole path is encrypted `/`s and all.
First file names are padded using using PKCS#7 to a multiple of 16
bytes before encryption.
They are then encrypted with EME using AES with 256 bit key. EME
(ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003
paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway.
This makes for determinstic encryption which is what we want - the
same filename must encrypt to the same thing.
This means that
* filenames with the same name will encrypt the same
* (though we can use directory flattening to avoid this if required)
* filenames which start the same won't have a common prefix
This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of
which are derived from the user password.
After encryption they are written out using a modified version of
standard `base32` encoding as described in RFC4648. The standard
encoding is modified in two ways:
* it becomes lower case (no-one likes upper case filenames!)
* we strip the padding character `=`
`base32` is used rather than the more efficient `base64` so rclone can be
used on case insensitive remotes (eg Windows, Amazon Drive).
### Key derivation ###
Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a fixed
salt to derive the 32+32+16 = 80 bytes of key material required.
`scrypt` makes it impractical to mount a dictionary attack on rclone
encrypted data.

1
fs/all/all.go
View file

@ -4,6 +4,7 @@ import (
// Active file systems // Active file systems
_ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/amazonclouddrive"
_ "github.com/ncw/rclone/b2" _ "github.com/ncw/rclone/b2"
_ "github.com/ncw/rclone/crypt"
_ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/googlecloudstorage"

15
fstest/fstests/fstests.go
View file

@ -8,6 +8,7 @@ package fstests
import ( import (
"bytes" "bytes"
"flag" "flag"
"fmt"
"io" "io"
"os" "os"
"path" "path"
@ -86,7 +87,7 @@ func TestInit(t *testing.T) {
t.Logf("Didn't find %q in config file - skipping tests", RemoteName) t.Logf("Didn't find %q in config file - skipping tests", RemoteName)
return return
} }
require.NoError(t, err) require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err))
fstest.TestMkdir(t, remote) fstest.TestMkdir(t, remote)
} }
@ -215,7 +216,7 @@ again:
tries++ tries++
goto again goto again
} }
require.NoError(t, err, "Put error") require.NoError(t, err, fmt.Sprintf("Put error: %v", err))
} }
file.Hashes = hash.Sums() file.Hashes = hash.Sums()
file.Check(t, obj, remote.Precision()) file.Check(t, obj, remote.Precision())
@ -335,7 +336,10 @@ func TestFsCopy(t *testing.T) {
// do the copy // do the copy
src := findObject(t, file1.Path) src := findObject(t, file1.Path)
dst, err := remote.(fs.Copier).Copy(src, file1Copy.Path) dst, err := remote.(fs.Copier).Copy(src, file1Copy.Path)
require.NoError(t, err) if err == fs.ErrorCantCopy {
t.Skip("FS can't copy")
}
require.NoError(t, err, fmt.Sprintf("Error: %#v", err))
// check file exists in new listing // check file exists in new listing
fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file1Copy}) fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file1Copy})
@ -365,6 +369,9 @@ func TestFsMove(t *testing.T) {
// do the move // do the move
src := findObject(t, file1.Path) src := findObject(t, file1.Path)
dst, err := remote.(fs.Mover).Move(src, file1Move.Path) dst, err := remote.(fs.Mover).Move(src, file1Move.Path)
if err == fs.ErrorCantMove {
t.Skip("FS can't move")
}
require.NoError(t, err) require.NoError(t, err)
// check file exists in new listing // check file exists in new listing
@ -521,7 +528,7 @@ func TestObjectOpen(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
hasher := fs.NewMultiHasher() hasher := fs.NewMultiHasher()
n, err := io.Copy(hasher, in) n, err := io.Copy(hasher, in)
require.NoError(t, err) require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err))
require.Equal(t, file1.Size, n, "Read wrong number of bytes") require.Equal(t, file1.Size, n, "Read wrong number of bytes")
err = in.Close() err = in.Close()
require.NoError(t, err) require.NoError(t, err)

4
fstest/fstests/gen_tests.go
View file

@ -61,7 +61,8 @@ import (
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests" "github.com/ncw/rclone/fstest/fstests"
"github.com/ncw/rclone/{{ .FsName }}" "github.com/ncw/rclone/{{ .FsName }}"
) {{ if eq .FsName "crypt" }} _ "github.com/ncw/rclone/local"
{{end}})
func init() { func init() {
fstests.NilObject = fs.Object((*{{ .FsName }}.Object)(nil)) fstests.NilObject = fs.Object((*{{ .FsName }}.Object)(nil))
@ -135,5 +136,6 @@ func main() {
generateTestProgram(t, fns, "Hubic") generateTestProgram(t, fns, "Hubic")
generateTestProgram(t, fns, "B2") generateTestProgram(t, fns, "B2")
generateTestProgram(t, fns, "Yandex") generateTestProgram(t, fns, "Yandex")
generateTestProgram(t, fns, "Crypt")
log.Printf("Done") log.Printf("Done")
} }