20da3e6352
* Add options to Put, PutUnchecked and Update for all Fses * Use these to create HashOption * Implement this in local * Pass the option in fs.Copy This has the effect that we only calculate hashes we need to in the local Fs which speeds up transfers significantly.
606 lines
16 KiB
Go
606 lines
16 KiB
Go
// Package crypt provides wrappers for Fs and Object which implement encryption
|
|
package crypt
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
// Globals
|
|
var (
|
|
// Flags
|
|
cryptShowMapping = fs.BoolP("crypt-show-mapping", "", false, "For all files listed show how the names encrypt.")
|
|
)
|
|
|
|
// Register with Fs
|
|
func init() {
|
|
fs.Register(&fs.RegInfo{
|
|
Name: "crypt",
|
|
Description: "Encrypt/Decrypt a remote",
|
|
NewFs: NewFs,
|
|
Options: []fs.Option{{
|
|
Name: "remote",
|
|
Help: "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
|
|
}, {
|
|
Name: "filename_encryption",
|
|
Help: "How to encrypt the filenames.",
|
|
Examples: []fs.OptionExample{
|
|
{
|
|
Value: "off",
|
|
Help: "Don't encrypt the file names. Adds a \".bin\" extension only.",
|
|
}, {
|
|
Value: "standard",
|
|
Help: "Encrypt the filenames see the docs for the details.",
|
|
}, {
|
|
Value: "obfuscate",
|
|
Help: "Very simple filename obfuscation.",
|
|
},
|
|
},
|
|
}, {
|
|
Name: "password",
|
|
Help: "Password or pass phrase for encryption.",
|
|
IsPassword: true,
|
|
}, {
|
|
Name: "password2",
|
|
Help: "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
|
|
IsPassword: true,
|
|
Optional: true,
|
|
}},
|
|
})
|
|
}
|
|
|
|
// NewFs contstructs an Fs from the path, container:path
|
|
func NewFs(name, rpath string) (fs.Fs, error) {
|
|
mode, err := NewNameEncryptionMode(fs.ConfigFileGet(name, "filename_encryption", "standard"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
password := fs.ConfigFileGet(name, "password", "")
|
|
if password == "" {
|
|
return nil, errors.New("password not set in config file")
|
|
}
|
|
password, err = fs.Reveal(password)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to decrypt password")
|
|
}
|
|
salt := fs.ConfigFileGet(name, "password2", "")
|
|
if salt != "" {
|
|
salt, err = fs.Reveal(salt)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to decrypt password2")
|
|
}
|
|
}
|
|
cipher, err := newCipher(mode, password, salt)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to make cipher")
|
|
}
|
|
remote := fs.ConfigFileGet(name, "remote")
|
|
if strings.HasPrefix(remote, name+":") {
|
|
return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
|
|
}
|
|
// Look for a file first
|
|
remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
|
|
wrappedFs, err := fs.NewFs(remotePath)
|
|
// if that didn't produce a file, look for a directory
|
|
if err != fs.ErrorIsFile {
|
|
remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
|
|
wrappedFs, err = fs.NewFs(remotePath)
|
|
}
|
|
if err != fs.ErrorIsFile && err != nil {
|
|
return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
|
|
}
|
|
f := &Fs{
|
|
Fs: wrappedFs,
|
|
name: name,
|
|
root: rpath,
|
|
cipher: cipher,
|
|
mode: mode,
|
|
}
|
|
// the features here are ones we could support, and they are
|
|
// ANDed with the ones from wrappedFs
|
|
f.features = (&fs.Features{
|
|
CaseInsensitive: mode == NameEncryptionOff,
|
|
DuplicateFiles: true,
|
|
ReadMimeType: false, // MimeTypes not supported with crypt
|
|
WriteMimeType: false,
|
|
}).Fill(f).Mask(wrappedFs)
|
|
return f, err
|
|
}
|
|
|
|
// Fs represents a wrapped fs.Fs
|
|
type Fs struct {
|
|
fs.Fs
|
|
name string
|
|
root string
|
|
features *fs.Features // optional features
|
|
cipher Cipher
|
|
mode NameEncryptionMode
|
|
}
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
func (f *Fs) Name() string {
|
|
return f.name
|
|
}
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
func (f *Fs) Root() string {
|
|
return f.root
|
|
}
|
|
|
|
// Features returns the optional features of this Fs
|
|
func (f *Fs) Features() *fs.Features {
|
|
return f.features
|
|
}
|
|
|
|
// String returns a description of the FS
|
|
func (f *Fs) String() string {
|
|
return fmt.Sprintf("Encrypted drive '%s:%s'", f.name, f.root)
|
|
}
|
|
|
|
// List the Fs into a channel
|
|
func (f *Fs) List(opts fs.ListOpts, dir string) {
|
|
f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir))
|
|
}
|
|
|
|
// NewObject finds the Object at remote.
|
|
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
|
o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.newObject(o), nil
|
|
}
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
//
|
|
// May create the object even if it returns an error - if so
|
|
// will return the object and the error, otherwise will return
|
|
// nil and the error
|
|
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
wrappedIn, err := f.cipher.EncryptData(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o, err := f.Fs.Put(wrappedIn, f.newObjectInfo(src))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.newObject(o), nil
|
|
}
|
|
|
|
// Hashes returns the supported hash sets.
|
|
func (f *Fs) Hashes() fs.HashSet {
|
|
return fs.HashSet(fs.HashNone)
|
|
}
|
|
|
|
// Mkdir makes the directory (container, bucket)
|
|
//
|
|
// Shouldn't return an error if it already exists
|
|
func (f *Fs) Mkdir(dir string) error {
|
|
return f.Fs.Mkdir(f.cipher.EncryptDirName(dir))
|
|
}
|
|
|
|
// Rmdir removes the directory (container, bucket) if empty
|
|
//
|
|
// Return an error if it doesn't exist or isn't empty
|
|
func (f *Fs) Rmdir(dir string) error {
|
|
return f.Fs.Rmdir(f.cipher.EncryptDirName(dir))
|
|
}
|
|
|
|
// Purge all files in the root and the root directory
|
|
//
|
|
// Implement this if you have a way of deleting all the files
|
|
// quicker than just running Remove() on the result of List()
|
|
//
|
|
// Return an error if it doesn't exist
|
|
func (f *Fs) Purge() error {
|
|
do := f.Fs.Features().Purge
|
|
if do == nil {
|
|
return fs.ErrorCantPurge
|
|
}
|
|
return do()
|
|
}
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
|
do := f.Fs.Features().Copy
|
|
if do == nil {
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
o, ok := src.(*Object)
|
|
if !ok {
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
oResult, err := do(o.Object, f.cipher.EncryptFileName(remote))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.newObject(oResult), nil
|
|
}
|
|
|
|
// Move src to this remote using server side move operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
|
do := f.Fs.Features().Move
|
|
if do == nil {
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
o, ok := src.(*Object)
|
|
if !ok {
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
oResult, err := do(o.Object, f.cipher.EncryptFileName(remote))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.newObject(oResult), nil
|
|
}
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
|
|
do := f.Fs.Features().DirMove
|
|
if do == nil {
|
|
return fs.ErrorCantDirMove
|
|
}
|
|
srcFs, ok := src.(*Fs)
|
|
if !ok {
|
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
|
return fs.ErrorCantDirMove
|
|
}
|
|
return do(srcFs.Fs, f.cipher.EncryptDirName(srcRemote), f.cipher.EncryptDirName(dstRemote))
|
|
}
|
|
|
|
// PutUnchecked uploads the object
|
|
//
|
|
// This will create a duplicate if we upload a new file without
|
|
// checking to see if there is one already - use Put() for that.
|
|
func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
do := f.Fs.Features().PutUnchecked
|
|
if do == nil {
|
|
return nil, errors.New("can't PutUnchecked")
|
|
}
|
|
wrappedIn, err := f.cipher.EncryptData(in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o, err := do(wrappedIn, f.newObjectInfo(src))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.newObject(o), nil
|
|
}
|
|
|
|
// CleanUp the trash in the Fs
|
|
//
|
|
// Implement this if you have a way of emptying the trash or
|
|
// otherwise cleaning up old versions of files.
|
|
func (f *Fs) CleanUp() error {
|
|
do := f.Fs.Features().CleanUp
|
|
if do == nil {
|
|
return errors.New("can't CleanUp")
|
|
}
|
|
return do()
|
|
}
|
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
func (f *Fs) UnWrap() fs.Fs {
|
|
return f.Fs
|
|
}
|
|
|
|
// ComputeHash takes the nonce from o, and encrypts the contents of
|
|
// src with it, and calcuates the hash given by HashType on the fly
|
|
//
|
|
// Note that we break lots of encapsulation in this function.
|
|
func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType fs.HashType) (hash string, err error) {
|
|
// Read the nonce - opening the file is sufficient to read the nonce in
|
|
in, err := o.Open()
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to read nonce")
|
|
}
|
|
nonce := in.(*decrypter).nonce
|
|
// fs.Debugf(o, "Read nonce % 2x", nonce)
|
|
|
|
// Check nonce isn't all zeros
|
|
isZero := true
|
|
for i := range nonce {
|
|
if nonce[i] != 0 {
|
|
isZero = false
|
|
}
|
|
}
|
|
if isZero {
|
|
fs.Errorf(o, "empty nonce read")
|
|
}
|
|
|
|
// Close in once we have read the nonce
|
|
err = in.Close()
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to close nonce read")
|
|
}
|
|
|
|
// Open the src for input
|
|
in, err = src.Open()
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to open src")
|
|
}
|
|
defer fs.CheckClose(in, &err)
|
|
|
|
// Now encrypt the src with the nonce
|
|
out, err := f.cipher.(*cipher).newEncrypter(in, &nonce)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to make encrypter")
|
|
}
|
|
|
|
// pipe into hash
|
|
m := fs.NewMultiHasher()
|
|
_, err = io.Copy(m, out)
|
|
if err != nil {
|
|
return "", errors.Wrap(err, "failed to hash data")
|
|
}
|
|
|
|
return m.Sums()[hashType], nil
|
|
}
|
|
|
|
// Object describes a wrapped for being read from the Fs
|
|
//
|
|
// This decrypts the remote name and decrypts the data
|
|
type Object struct {
|
|
fs.Object
|
|
f *Fs
|
|
}
|
|
|
|
func (f *Fs) newObject(o fs.Object) *Object {
|
|
return &Object{
|
|
Object: o,
|
|
f: f,
|
|
}
|
|
}
|
|
|
|
// Fs returns read only access to the Fs that this object is part of
|
|
func (o *Object) Fs() fs.Info {
|
|
return o.f
|
|
}
|
|
|
|
// Return a string version
|
|
func (o *Object) String() string {
|
|
if o == nil {
|
|
return "<nil>"
|
|
}
|
|
return o.Remote()
|
|
}
|
|
|
|
// Remote returns the remote path
|
|
func (o *Object) Remote() string {
|
|
remote := o.Object.Remote()
|
|
decryptedName, err := o.f.cipher.DecryptFileName(remote)
|
|
if err != nil {
|
|
fs.Debugf(remote, "Undecryptable file name: %v", err)
|
|
return remote
|
|
}
|
|
return decryptedName
|
|
}
|
|
|
|
// Size returns the size of the file
|
|
func (o *Object) Size() int64 {
|
|
size, err := o.f.cipher.DecryptedSize(o.Object.Size())
|
|
if err != nil {
|
|
fs.Debugf(o, "Bad size for decrypt: %v", err)
|
|
}
|
|
return size
|
|
}
|
|
|
|
// Hash returns the selected checksum of the file
|
|
// If no checksum is available it returns ""
|
|
func (o *Object) Hash(hash fs.HashType) (string, error) {
|
|
return "", nil
|
|
}
|
|
|
|
// UnWrap returns the wrapped Object
|
|
func (o *Object) UnWrap() fs.Object {
|
|
return o.Object
|
|
}
|
|
|
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
|
func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
|
var offset int64
|
|
for _, option := range options {
|
|
switch x := option.(type) {
|
|
case *fs.SeekOption:
|
|
offset = x.Offset
|
|
default:
|
|
if option.Mandatory() {
|
|
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
|
}
|
|
}
|
|
}
|
|
rc, err = o.f.cipher.DecryptDataSeek(func(underlyingOffset int64) (io.ReadCloser, error) {
|
|
if underlyingOffset == 0 {
|
|
// Open with no seek
|
|
return o.Object.Open()
|
|
}
|
|
// Open stream with a seek of underlyingOffset
|
|
return o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
|
|
}, offset)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rc, err
|
|
}
|
|
|
|
// Update in to the object with the modTime given of the given size
|
|
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
wrappedIn, err := o.f.cipher.EncryptData(in)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return o.Object.Update(wrappedIn, o.f.newObjectInfo(src))
|
|
}
|
|
|
|
// newDir returns a dir with the Name decrypted
|
|
func (f *Fs) newDir(dir *fs.Dir) *fs.Dir {
|
|
new := *dir
|
|
remote := dir.Name
|
|
decryptedRemote, err := f.cipher.DecryptDirName(remote)
|
|
if err != nil {
|
|
fs.Debugf(remote, "Undecryptable dir name: %v", err)
|
|
} else {
|
|
new.Name = decryptedRemote
|
|
}
|
|
return &new
|
|
}
|
|
|
|
// ObjectInfo describes a wrapped fs.ObjectInfo for being the source
|
|
//
|
|
// This encrypts the remote name and adjusts the size
|
|
type ObjectInfo struct {
|
|
fs.ObjectInfo
|
|
f *Fs
|
|
}
|
|
|
|
func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo {
|
|
return &ObjectInfo{
|
|
ObjectInfo: src,
|
|
f: f,
|
|
}
|
|
}
|
|
|
|
// Fs returns read only access to the Fs that this object is part of
|
|
func (o *ObjectInfo) Fs() fs.Info {
|
|
return o.f
|
|
}
|
|
|
|
// Remote returns the remote path
|
|
func (o *ObjectInfo) Remote() string {
|
|
return o.f.cipher.EncryptFileName(o.ObjectInfo.Remote())
|
|
}
|
|
|
|
// Size returns the size of the file
|
|
func (o *ObjectInfo) Size() int64 {
|
|
return o.f.cipher.EncryptedSize(o.ObjectInfo.Size())
|
|
}
|
|
|
|
// Hash returns the selected checksum of the file
|
|
// If no checksum is available it returns ""
|
|
func (o *ObjectInfo) Hash(hash fs.HashType) (string, error) {
|
|
return "", nil
|
|
}
|
|
|
|
// ListOpts wraps a listopts decrypting the directory listing and
|
|
// replacing the Objects
|
|
type ListOpts struct {
|
|
fs.ListOpts
|
|
f *Fs
|
|
dir string // dir we are listing
|
|
mu sync.Mutex // to protect dirs
|
|
dirs map[string]struct{} // keep track of synthetic directory objects added
|
|
}
|
|
|
|
// Make a ListOpts wrapper
|
|
func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts {
|
|
if dir != "" {
|
|
dir += "/"
|
|
}
|
|
return &ListOpts{
|
|
ListOpts: lo,
|
|
f: f,
|
|
dir: dir,
|
|
dirs: make(map[string]struct{}),
|
|
}
|
|
|
|
}
|
|
|
|
// Level gets the recursion level for this listing.
|
|
//
|
|
// Fses may ignore this, but should implement it for improved efficiency if possible.
|
|
//
|
|
// Level 1 means list just the contents of the directory
|
|
//
|
|
// Each returned item must have less than level `/`s in.
|
|
func (lo *ListOpts) Level() int {
|
|
return lo.ListOpts.Level()
|
|
}
|
|
|
|
// Add an object to the output.
|
|
// If the function returns true, the operation has been aborted.
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
|
|
remote := obj.Remote()
|
|
decryptedRemote, err := lo.f.cipher.DecryptFileName(remote)
|
|
if err != nil {
|
|
fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
|
|
return lo.ListOpts.IsFinished()
|
|
}
|
|
if *cryptShowMapping {
|
|
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
|
|
}
|
|
return lo.ListOpts.Add(lo.f.newObject(obj))
|
|
}
|
|
|
|
// AddDir adds a directory to the output.
|
|
// If the function returns true, the operation has been aborted.
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
|
|
remote := dir.Name
|
|
decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
|
|
if err != nil {
|
|
fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
|
|
return lo.ListOpts.IsFinished()
|
|
}
|
|
if *cryptShowMapping {
|
|
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
|
|
}
|
|
return lo.ListOpts.AddDir(lo.f.newDir(dir))
|
|
}
|
|
|
|
// IncludeDirectory returns whether this directory should be
|
|
// included in the listing (and recursed into or not).
|
|
func (lo *ListOpts) IncludeDirectory(remote string) bool {
|
|
decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
|
|
if err != nil {
|
|
fs.Debugf(remote, "Not including undecryptable directory name: %v", err)
|
|
return false
|
|
}
|
|
return lo.ListOpts.IncludeDirectory(decryptedRemote)
|
|
}
|
|
|
|
// Check the interfaces are satisfied
|
|
var (
|
|
_ fs.Fs = (*Fs)(nil)
|
|
_ fs.Purger = (*Fs)(nil)
|
|
_ fs.Copier = (*Fs)(nil)
|
|
_ fs.Mover = (*Fs)(nil)
|
|
_ fs.DirMover = (*Fs)(nil)
|
|
_ fs.PutUncheckeder = (*Fs)(nil)
|
|
_ fs.CleanUpper = (*Fs)(nil)
|
|
_ fs.UnWrapper = (*Fs)(nil)
|
|
_ fs.ObjectInfo = (*ObjectInfo)(nil)
|
|
_ fs.Object = (*Object)(nil)
|
|
_ fs.ListOpts = (*ListOpts)(nil)
|
|
)
|