Move all backends into backend directory
This commit is contained in:
Nick Craig-Wood
parent
0a7731cf0d
commit
b8b620f5c2
143 changed files with 88 additions and 87 deletions
927
backend/local/local.go
Normal file
927
backend/local/local.go
Normal file
|
|
@ -0,0 +1,927 @@
|
|||
// Package local provides a filesystem interface
|
||||
package local
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/appengine/log"
|
||||
)
|
||||
|
||||
var (
|
||||
followSymlinks = fs.BoolP("copy-links", "L", false, "Follow symlinks and copy the pointed to item.")
|
||||
skipSymlinks = fs.BoolP("skip-links", "", false, "Don't warn about skipped symlinks.")
|
||||
noUTFNorm = fs.BoolP("local-no-unicode-normalization", "", false, "Don't apply unicode normalization to paths and filenames")
|
||||
)
|
||||
|
||||
// Constants
|
||||
const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fsi := &fs.RegInfo{
|
||||
Name: "local",
|
||||
Description: "Local Disk",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "nounc",
|
||||
Help: "Disable UNC (long path names) conversion on Windows",
|
||||
Optional: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "true",
|
||||
Help: "Disables long file names",
|
||||
}},
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Fs represents a local filesystem rooted at root
|
||||
type Fs struct {
|
||||
name string // the name of the remote
|
||||
root string // The root directory (OS path)
|
||||
features *fs.Features // optional features
|
||||
dev uint64 // device number of root node
|
||||
precisionOk sync.Once // Whether we need to read the precision
|
||||
precision time.Duration // precision of local filesystem
|
||||
wmu sync.Mutex // used for locking access to 'warned'.
|
||||
warned map[string]struct{} // whether we have warned about this string
|
||||
nounc bool // Skip UNC conversion on Windows
|
||||
// do os.Lstat or os.Stat
|
||||
lstat func(name string) (os.FileInfo, error)
|
||||
dirNames *mapper // directory name mapping
|
||||
}
|
||||
|
||||
// Object represents a local filesystem object
|
||||
type Object struct {
|
||||
fs *Fs // The Fs this object is part of
|
||||
remote string // The remote path - properly UTF-8 encoded - for rclone
|
||||
path string // The local path - may not be properly UTF-8 encoded - for OS
|
||||
size int64 // file metadata - always present
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
hashes map[fs.HashType]string // Hashes
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// NewFs constructs an Fs from the path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
var err error
|
||||
|
||||
if *noUTFNorm {
|
||||
log.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
|
||||
}
|
||||
|
||||
nounc := fs.ConfigFileGet(name, "nounc")
|
||||
f := &Fs{
|
||||
name: name,
|
||||
warned: make(map[string]struct{}),
|
||||
nounc: nounc == "true",
|
||||
dev: devUnset,
|
||||
lstat: os.Lstat,
|
||||
dirNames: newMapper(),
|
||||
}
|
||||
f.root = f.cleanPath(root)
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: f.caseInsensitive(),
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(f)
|
||||
if *followSymlinks {
|
||||
f.lstat = os.Stat
|
||||
}
|
||||
|
||||
// Check to see if this points to a file
|
||||
fi, err := f.lstat(f.root)
|
||||
if err == nil {
|
||||
f.dev = readDevice(fi)
|
||||
}
|
||||
if err == nil && fi.Mode().IsRegular() {
|
||||
// It is a file, so use the parent as the root
|
||||
f.root, _ = getDirFile(f.root)
|
||||
// return an error with an fs which points to the parent
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("Local file system at %s", f.root)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// caseInsenstive returns whether the remote is case insensitive or not
|
||||
func (f *Fs) caseInsensitive() bool {
|
||||
// FIXME not entirely accurate since you can have case
|
||||
// sensitive Fses on darwin and case insenstive Fses on linux.
|
||||
// Should probably check but that would involve creating a
|
||||
// file in the remote to be most accurate which probably isn't
|
||||
// desirable.
|
||||
return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
|
||||
}
|
||||
|
||||
// newObject makes a half completed Object
|
||||
//
|
||||
// if dstPath is empty then it is made from remote
|
||||
func (f *Fs) newObject(remote, dstPath string) *Object {
|
||||
if dstPath == "" {
|
||||
dstPath = f.cleanPath(filepath.Join(f.root, remote))
|
||||
}
|
||||
remote = f.cleanRemote(remote)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
path: dstPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Return an Object from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Object, error) {
|
||||
o := f.newObject(remote, dstPath)
|
||||
if info != nil {
|
||||
o.setMetadata(info)
|
||||
} else {
|
||||
err := o.lstat()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if o.mode.IsDir() {
|
||||
return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
return f.newObjectWithInfo(remote, "", nil)
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
dir = f.dirNames.Load(dir)
|
||||
fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
|
||||
remote := f.cleanRemote(dir)
|
||||
_, err = os.Stat(fsDirPath)
|
||||
if err != nil {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
fd, err := os.Open(fsDirPath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to open directory %q", dir)
|
||||
}
|
||||
defer func() {
|
||||
cerr := fd.Close()
|
||||
if cerr != nil && err == nil {
|
||||
err = errors.Wrapf(cerr, "failed to close directory %q:", dir)
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
fis, err := fd.Readdir(1024)
|
||||
if err == io.EOF && len(fis) == 0 {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to read directory %q", dir)
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
name := fi.Name()
|
||||
mode := fi.Mode()
|
||||
newRemote := path.Join(remote, name)
|
||||
newPath := filepath.Join(fsDirPath, name)
|
||||
// Follow symlinks if required
|
||||
if *followSymlinks && (mode&os.ModeSymlink) != 0 {
|
||||
fi, err = os.Stat(newPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode = fi.Mode()
|
||||
}
|
||||
if fi.IsDir() {
|
||||
// Ignore directories which are symlinks. These are junction points under windows which
|
||||
// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
|
||||
if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi) {
|
||||
d := fs.NewDir(f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), fi.ModTime())
|
||||
entries = append(entries, d)
|
||||
}
|
||||
} else {
|
||||
fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fso.Storable() {
|
||||
entries = append(entries, fso)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// cleanRemote makes string a valid UTF-8 string for remote strings.
|
||||
//
|
||||
// Any invalid UTF-8 characters will be replaced with utf8.RuneError
|
||||
// It also normalises the UTF-8 and converts the slashes if necessary.
|
||||
func (f *Fs) cleanRemote(name string) string {
|
||||
if !utf8.ValidString(name) {
|
||||
f.wmu.Lock()
|
||||
if _, ok := f.warned[name]; !ok {
|
||||
fs.Logf(f, "Replacing invalid UTF-8 characters in %q", name)
|
||||
f.warned[name] = struct{}{}
|
||||
}
|
||||
f.wmu.Unlock()
|
||||
name = string([]rune(name))
|
||||
}
|
||||
name = filepath.ToSlash(name)
|
||||
return name
|
||||
}
|
||||
|
||||
// mapper maps raw to cleaned directory names
|
||||
type mapper struct {
|
||||
mu sync.RWMutex // mutex to protect the below
|
||||
m map[string]string // map of un-normalised directory names
|
||||
}
|
||||
|
||||
func newMapper() *mapper {
|
||||
return &mapper{
|
||||
m: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup a directory name to make a local name (reverses
|
||||
// cleanDirName)
|
||||
//
|
||||
// FIXME this is temporary before we make a proper Directory object
|
||||
func (m *mapper) Load(in string) string {
|
||||
m.mu.RLock()
|
||||
out, ok := m.m[in]
|
||||
m.mu.RUnlock()
|
||||
if ok {
|
||||
return out
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Cleans a directory name recording if it needed to be altered
|
||||
//
|
||||
// FIXME this is temporary before we make a proper Directory object
|
||||
func (m *mapper) Save(in, out string) string {
|
||||
if in != out {
|
||||
m.mu.Lock()
|
||||
m.m[out] = in
|
||||
m.mu.Unlock()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Put the Object to the local filesystem
|
||||
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
// Temporary Object under construction - info filled in by Update()
|
||||
o := f.newObject(remote, "")
|
||||
err := o.Update(in, src, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||||
func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
return f.Put(in, src, options...)
|
||||
}
|
||||
|
||||
// Mkdir creates the directory if it doesn't exist
|
||||
func (f *Fs) Mkdir(dir string) error {
|
||||
// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
|
||||
root := f.cleanPath(filepath.Join(f.root, dir))
|
||||
err := os.MkdirAll(root, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir == "" {
|
||||
fi, err := f.lstat(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.dev = readDevice(fi)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rmdir removes the directory
|
||||
//
|
||||
// If it isn't empty it will return an error
|
||||
func (f *Fs) Rmdir(dir string) error {
|
||||
root := f.cleanPath(filepath.Join(f.root, dir))
|
||||
return os.Remove(root)
|
||||
}
|
||||
|
||||
// Precision of the file system
|
||||
func (f *Fs) Precision() (precision time.Duration) {
|
||||
f.precisionOk.Do(func() {
|
||||
f.precision = f.readPrecision()
|
||||
})
|
||||
return f.precision
|
||||
}
|
||||
|
||||
// Read the precision
|
||||
func (f *Fs) readPrecision() (precision time.Duration) {
|
||||
// Default precision of 1s
|
||||
precision = time.Second
|
||||
|
||||
// Create temporary file and test it
|
||||
fd, err := ioutil.TempFile("", "rclone")
|
||||
if err != nil {
|
||||
// If failed return 1s
|
||||
// fmt.Println("Failed to create temp file", err)
|
||||
return time.Second
|
||||
}
|
||||
path := fd.Name()
|
||||
// fmt.Println("Created temp file", path)
|
||||
err = fd.Close()
|
||||
if err != nil {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Delete it on return
|
||||
defer func() {
|
||||
// fmt.Println("Remove temp file")
|
||||
_ = os.Remove(path) // ignore error
|
||||
}()
|
||||
|
||||
// Find the minimum duration we can detect
|
||||
for duration := time.Duration(1); duration < time.Second; duration *= 10 {
|
||||
// Current time with delta
|
||||
t := time.Unix(time.Now().Unix(), int64(duration))
|
||||
err := os.Chtimes(path, t, t)
|
||||
if err != nil {
|
||||
// fmt.Println("Failed to Chtimes", err)
|
||||
break
|
||||
}
|
||||
|
||||
// Read the actual time back
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
// fmt.Println("Failed to Stat", err)
|
||||
break
|
||||
}
|
||||
|
||||
// If it matches - have found the precision
|
||||
// fmt.Println("compare", fi.ModTime(), t)
|
||||
if fi.ModTime() == t {
|
||||
// fmt.Println("Precision detected as", duration)
|
||||
return duration
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Purge deletes all the files and directories
|
||||
//
|
||||
// Optional interface: Only implement this if you have a way of
|
||||
// deleting all the files quicker than just running Remove() on the
|
||||
// result of List()
|
||||
func (f *Fs) Purge() error {
|
||||
fi, err := f.lstat(f.root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !fi.Mode().IsDir() {
|
||||
return errors.Errorf("can't purge non directory: %q", f.root)
|
||||
}
|
||||
return os.RemoveAll(f.root)
|
||||
}
|
||||
|
||||
// Move src to this remote using server side move operations.
|
||||
//
|
||||
// This is stored with the remote path given
|
||||
//
|
||||
// It returns the destination Object and a possible error
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantMove
|
||||
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
// Temporary Object under construction
|
||||
dstObj := f.newObject(remote, "")
|
||||
|
||||
// Check it is a file if it exists
|
||||
err := dstObj.lstat()
|
||||
if os.IsNotExist(err) {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else if !dstObj.mode.IsRegular() {
|
||||
// It isn't a file
|
||||
return nil, errors.New("can't move file onto non-file")
|
||||
}
|
||||
|
||||
// Create destination
|
||||
err = dstObj.mkdirAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
err = os.Rename(srcObj.path, dstObj.path)
|
||||
if os.IsNotExist(err) {
|
||||
// race condition, source was deleted in the meantime
|
||||
return nil, err
|
||||
} else if os.IsPermission(err) {
|
||||
// not enough rights to write to dst
|
||||
return nil, err
|
||||
} else if err != nil {
|
||||
// not quite clear, but probably trying to move a file across file system
|
||||
// boundaries. Copying might still work.
|
||||
fs.Errorf(src, "Can't move: %v: trying copy", err)
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
// Update the info
|
||||
err = dstObj.lstat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server side move operations.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantDirMove
|
||||
//
|
||||
// If destination exists then return fs.ErrorDirExists
|
||||
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := f.cleanPath(filepath.Join(srcFs.root, srcRemote))
|
||||
dstPath := f.cleanPath(filepath.Join(f.root, dstRemote))
|
||||
|
||||
// Check if destination exists
|
||||
_, err := os.Lstat(dstPath)
|
||||
if !os.IsNotExist(err) {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
|
||||
// Create parent of destination
|
||||
dstParentPath, _ := getDirFile(dstPath)
|
||||
err = os.MkdirAll(dstParentPath, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
return os.Rename(srcPath, dstPath)
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() fs.HashSet {
|
||||
return fs.SupportedHashes
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Fs returns the parent Fs
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Hash returns the requested hash of a file as a lowercase hex string
|
||||
func (o *Object) Hash(r fs.HashType) (string, error) {
|
||||
// Check that the underlying file hasn't changed
|
||||
oldtime := o.modTime
|
||||
oldsize := o.size
|
||||
err := o.lstat()
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "hash: failed to stat")
|
||||
}
|
||||
|
||||
if !o.modTime.Equal(oldtime) || oldsize != o.size {
|
||||
o.hashes = nil
|
||||
}
|
||||
|
||||
if o.hashes == nil {
|
||||
o.hashes = make(map[fs.HashType]string)
|
||||
in, err := os.Open(o.path)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "hash: failed to open")
|
||||
}
|
||||
o.hashes, err = fs.HashStream(in)
|
||||
closeErr := in.Close()
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "hash: failed to read")
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", errors.Wrap(closeErr, "hash: failed to close")
|
||||
}
|
||||
}
|
||||
return o.hashes[r], nil
|
||||
}
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (o *Object) Size() int64 {
|
||||
return o.size
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
func (o *Object) ModTime() time.Time {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(modTime time.Time) error {
|
||||
err := os.Chtimes(o.path, modTime, modTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Re-read metadata
|
||||
return o.lstat()
|
||||
}
|
||||
|
||||
// Storable returns a boolean showing if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
// Check for control characters in the remote name and show non storable
|
||||
for _, c := range o.Remote() {
|
||||
if c >= 0x00 && c < 0x20 || c == 0x7F {
|
||||
fs.Logf(o.fs, "Can't store file with control characters: %q", o.Remote())
|
||||
return false
|
||||
}
|
||||
}
|
||||
mode := o.mode
|
||||
// On windows a file with os.ModeSymlink represents a file with reparse points
|
||||
if runtime.GOOS == "windows" && (mode&os.ModeSymlink) != 0 {
|
||||
fs.Debugf(o, "Clearing symlink bit to allow a file with reparse points to be copied")
|
||||
mode &^= os.ModeSymlink
|
||||
}
|
||||
if mode&os.ModeSymlink != 0 {
|
||||
if !*skipSymlinks {
|
||||
fs.Logf(o, "Can't follow symlink without -L/--copy-links")
|
||||
}
|
||||
return false
|
||||
} else if mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
||||
fs.Logf(o, "Can't transfer non file/directory")
|
||||
return false
|
||||
} else if mode&os.ModeDir != 0 {
|
||||
// fs.Debugf(o, "Skipping directory")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// localOpenFile wraps an io.ReadCloser and updates the md5sum of the
|
||||
// object that is read
|
||||
type localOpenFile struct {
|
||||
o *Object // object that is open
|
||||
in io.ReadCloser // handle we are wrapping
|
||||
hash *fs.MultiHasher // currently accumulating hashes
|
||||
}
|
||||
|
||||
// Read bytes from the object - see io.Reader
|
||||
func (file *localOpenFile) Read(p []byte) (n int, err error) {
|
||||
n, err = file.in.Read(p)
|
||||
if n > 0 {
|
||||
// Hash routines never return an error
|
||||
_, _ = file.hash.Write(p[:n])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Close the object and update the hashes
|
||||
func (file *localOpenFile) Close() (err error) {
|
||||
err = file.in.Close()
|
||||
if err == nil {
|
||||
if file.hash.Size() == file.o.Size() {
|
||||
file.o.hashes = file.hash.Sums()
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
var offset int64
|
||||
hashes := fs.SupportedHashes
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.HashesOption:
|
||||
hashes = x.Hashes
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fd, err := os.Open(o.path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if offset != 0 {
|
||||
// seek the object
|
||||
_, err = fd.Seek(offset, 0)
|
||||
// don't attempt to make checksums
|
||||
return fd, err
|
||||
}
|
||||
hash, err := fs.NewMultiHasherTypes(hashes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Update the md5sum as we go along
|
||||
in = &localOpenFile{
|
||||
o: o,
|
||||
in: fd,
|
||||
hash: hash,
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// mkdirAll makes all the directories needed to store the object
|
||||
func (o *Object) mkdirAll() error {
|
||||
dir, _ := getDirFile(o.path)
|
||||
return os.MkdirAll(dir, 0777)
|
||||
}
|
||||
|
||||
// Update the object from in with modTime and size
|
||||
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
hashes := fs.SupportedHashes
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.HashesOption:
|
||||
hashes = x.Hashes
|
||||
}
|
||||
}
|
||||
|
||||
err := o.mkdirAll()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Calculate the hash of the object we are reading as we go along
|
||||
hash, err := fs.NewMultiHasherTypes(hashes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
in = io.TeeReader(in, hash)
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
fs.Logf(o, "Removing partially written file on error: %v", err)
|
||||
if removeErr := os.Remove(o.path); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// All successful so update the hashes
|
||||
o.hashes = hash.Sums()
|
||||
|
||||
// Set the mtime
|
||||
err = o.SetModTime(src.ModTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ReRead info now that we have finished
|
||||
return o.lstat()
|
||||
}
|
||||
|
||||
// setMetadata sets the file info from the os.FileInfo passed in
|
||||
func (o *Object) setMetadata(info os.FileInfo) {
|
||||
// Don't overwrite the info if we don't need to
|
||||
// this avoids upsetting the race detector
|
||||
if o.size != info.Size() {
|
||||
o.size = info.Size()
|
||||
}
|
||||
if !o.modTime.Equal(info.ModTime()) {
|
||||
o.modTime = info.ModTime()
|
||||
}
|
||||
if o.mode != info.Mode() {
|
||||
o.mode = info.Mode()
|
||||
}
|
||||
}
|
||||
|
||||
// Stat a Object into info
|
||||
func (o *Object) lstat() error {
|
||||
info, err := o.fs.lstat(o.path)
|
||||
if err == nil {
|
||||
o.setMetadata(info)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove() error {
|
||||
return os.Remove(o.path)
|
||||
}
|
||||
|
||||
// Return the directory and file from an OS path. Assumes
|
||||
// os.PathSeparator is used.
|
||||
func getDirFile(s string) (string, string) {
|
||||
i := strings.LastIndex(s, string(os.PathSeparator))
|
||||
dir, file := s[:i], s[i+1:]
|
||||
if dir == "" {
|
||||
dir = string(os.PathSeparator)
|
||||
}
|
||||
return dir, file
|
||||
}
|
||||
|
||||
// cleanPathFragment cleans an OS path fragment which is part of a
|
||||
// bigger path and not necessarily absolute
|
||||
func cleanPathFragment(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
s = filepath.Clean(s)
|
||||
if runtime.GOOS == "windows" {
|
||||
s = strings.Replace(s, `/`, `\`, -1)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// cleanPath cleans and makes absolute the path passed in and returns
|
||||
// an OS path.
|
||||
//
|
||||
// The input might be in OS form or rclone form or a mixture, but the
|
||||
// output is in OS form.
|
||||
//
|
||||
// On windows it makes the path UNC also and replaces any characters
|
||||
// Windows can't deal with with their replacements.
|
||||
func (f *Fs) cleanPath(s string) string {
|
||||
s = cleanPathFragment(s)
|
||||
if runtime.GOOS == "windows" {
|
||||
if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
|
||||
s2, err := filepath.Abs(s)
|
||||
if err == nil {
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
if !f.nounc {
|
||||
// Convert to UNC
|
||||
s = uncPath(s)
|
||||
}
|
||||
s = cleanWindowsName(f, s)
|
||||
} else {
|
||||
if !filepath.IsAbs(s) {
|
||||
s2, err := filepath.Abs(s)
|
||||
if err == nil {
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Pattern to match a windows absolute path: "c:\" and similar
|
||||
var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
|
||||
|
||||
// uncPath converts an absolute Windows path
|
||||
// to a UNC long path.
|
||||
func uncPath(s string) string {
|
||||
// UNC can NOT use "/", so convert all to "\"
|
||||
s = strings.Replace(s, `/`, `\`, -1)
|
||||
|
||||
// If prefix is "\\", we already have a UNC path or server.
|
||||
if strings.HasPrefix(s, `\\`) {
|
||||
// If already long path, just keep it
|
||||
if strings.HasPrefix(s, `\\?\`) {
|
||||
return s
|
||||
}
|
||||
|
||||
// Trim "\\" from path and add UNC prefix.
|
||||
return `\\?\UNC\` + strings.TrimPrefix(s, `\\`)
|
||||
}
|
||||
if isAbsWinDrive.MatchString(s) {
|
||||
return `\\?\` + s
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// cleanWindowsName will clean invalid Windows characters replacing them with _
|
||||
func cleanWindowsName(f *Fs, name string) string {
|
||||
original := name
|
||||
var name2 string
|
||||
if strings.HasPrefix(name, `\\?\`) {
|
||||
name2 = `\\?\`
|
||||
name = strings.TrimPrefix(name, `\\?\`)
|
||||
}
|
||||
if strings.HasPrefix(name, `//?/`) {
|
||||
name2 = `//?/`
|
||||
name = strings.TrimPrefix(name, `//?/`)
|
||||
}
|
||||
// Colon is allowed as part of a drive name X:\
|
||||
colonAt := strings.Index(name, ":")
|
||||
if colonAt > 0 && colonAt < 3 && len(name) > colonAt+1 {
|
||||
// Copy to name2, which is unfiltered
|
||||
name2 += name[0 : colonAt+1]
|
||||
name = name[colonAt+1:]
|
||||
}
|
||||
|
||||
name2 += strings.Map(func(r rune) rune {
|
||||
switch r {
|
||||
case '<', '>', '"', '|', '?', '*', ':':
|
||||
return '_'
|
||||
}
|
||||
return r
|
||||
}, name)
|
||||
|
||||
if name2 != original && f != nil {
|
||||
f.wmu.Lock()
|
||||
if _, ok := f.warned[name]; !ok {
|
||||
fs.Logf(f, "Replacing invalid characters in %q to %q", name, name2)
|
||||
f.warned[name] = struct{}{}
|
||||
}
|
||||
f.wmu.Unlock()
|
||||
}
|
||||
return name2
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Purger = &Fs{}
|
||||
_ fs.PutStreamer = &Fs{}
|
||||
_ fs.Mover = &Fs{}
|
||||
_ fs.DirMover = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue