rclone/backend/adb/adb.go

package adb

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/config/configstruct"
	"github.com/ncw/rclone/fs/hash"
	"github.com/ncw/rclone/lib/readers"
	"github.com/pkg/errors"
	adb "github.com/thinkhy/go-adb"
	"github.com/thinkhy/go-adb/wire"
)

// Register with Fs
func init() {
	fs.Register(&fs.RegInfo{
		Name:        "adb",
		Description: "Android Debug Bridge",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "serial",
			Help:     "The device serial to use. Leave empty for auto selection.",
			Advanced: true,
		}, {
			Name:     "host",
			Default:  "localhost",
			Help:     "The ADB server host.",
			Advanced: true,
		}, {
			Name:     "port",
			Default:  5037,
			Help:     "The ADB server port.",
			Advanced: true,
		}, {
			Name:     "executable",
			Help:     "The ADB executable path.",
			Advanced: true,
		}, {
			Name:     "copy_links",
			Help:     "Follow symlinks and copy the pointed to item.",
			Default:  false,
			Advanced: true,
		}},
	})
}

// Options defines the configuration for this backend
type Options struct {
	Serial         string
	Host           string
	Port           uint16
	Executable     string
	FollowSymlinks bool `config:"copy_links"`
}

// Fs represents a adb device
type Fs struct {
	name        string       // name of this remote
	root        string       // the path we are working on
	opt         Options      // parsed options
	features    *fs.Features // optional features
	client      *adb.Adb
	device      *execDevice
	statFunc    statFunc
	statFuncMu  sync.Mutex
	touchFunc   touchFunc
	touchFuncMu sync.Mutex
}

// Object describes a adb file
type Object struct {
	fs      *Fs    // what this object is part of
	remote  string // The remote path
	size    int64
	mode    os.FileMode
	modTime time.Time
}

// ------------------------------------------------------------

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("ADB root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
	// Parse config into Options struct
	opt := new(Options)
	err := configstruct.Set(m, opt)
	if err != nil {
		return nil, err
	}

	if root == "" {
		root = "/"
	}

	f := &Fs{
		name:      name,
		root:      root,
		opt:       *opt,
		statFunc:  (*Object).statTry,
		touchFunc: (*Object).touchTry,
	}
	f.features = (&fs.Features{
		CanHaveEmptyDirectories: true,
	}).Fill(f)

	f.client, err = adb.NewWithConfig(adb.ServerConfig{
		Host:      opt.Host,
		Port:      int(opt.Port),
		PathToAdb: opt.Executable,
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Could not configure ADB server")
	}
	err = f.client.StartServer()
	if err != nil {
		return nil, errors.Wrapf(err, "Could not start ADB server")
	}

	serverVersion, err := f.client.ServerVersion()
	if err != nil {
		return nil, errors.Wrapf(err, "Could not get ADB server version")
	}
	fs.Debugf(f, "ADB server version: 0x%X", serverVersion)

	serials, err := f.client.ListDeviceSerials()
	if err != nil {
		return nil, errors.Wrapf(err, "Could not get ADB devices")
	}
	descriptor := adb.AnyDevice()
	if opt.Serial != "" {
		descriptor = adb.DeviceWithSerial(opt.Serial)
	}
	if len(serials) > 1 && opt.Serial == "" {
		return nil, errors.New("Multiple ADB devices found. Use the serial config to select a specific device")
	}
	f.device = &execDevice{f.client.Device(descriptor)}

	// follow symlinks for root pathes
	entry, err := f.newEntryFollowSymlinks("")
	switch err {
	case nil:
	case fs.ErrorObjectNotFound:
	default:
		return nil, err
	}
	switch entry.(type) {
	case fs.Object:
		f.root = path.Dir(f.root)
		return f, fs.ErrorIsFile
	case nil:
		return f, nil
	case fs.Directory:
		return f, nil
	default:
		return nil, errors.Errorf("Invalid root entry type %t", entry)
	}
}

// Precision of the object storage system
func (f *Fs) Precision() time.Duration {
	return 1 * time.Second
}

// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
	return hash.Set(hash.None)
}

// List the objects and directories in dir into entries.  The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
	p := path.Join(f.root, dir)
	dirEntries, err := f.device.ListDirEntries(p)
	if err != nil {
		return nil, errors.Wrap(err, "ListDirEntries")
	}

	defer fs.CheckClose(dirEntries, &err)

	found := false
	for dirEntries.Next() {
		found = true
		dirEntry := dirEntries.Entry()
		switch dirEntry.Name {
		case ".", "..":
			continue
		}
		fsEntry, err := f.entryForDirEntry(path.Join(dir, dirEntry.Name), dirEntry, f.opt.FollowSymlinks)
		if err != nil {
			fs.Errorf(p, "Listing error: %q: %v", dirEntry.Name, err)
			return nil, err
		} else if fsEntry != nil {
			entries = append(entries, fsEntry)
		} else {
			fs.Debugf(f, "Skipping DirEntry %#v", dirEntry)
		}
	}
	err = dirEntries.Err()
	if err != nil {
		return nil, errors.Wrap(err, "ListDirEntries")
	}
	if !found {
		return nil, fs.ErrorDirNotFound
	}
	return
}

func (f *Fs) entryForDirEntry(remote string, e *adb.DirEntry, followSymlinks bool) (fs.DirEntry, error) {
	o := f.newObjectWithInfo(remote, e)
	// Follow symlinks if required
	if followSymlinks && (e.Mode&os.ModeSymlink) != 0 {
		err := f.statFunc(&o)
		if err != nil {
			return nil, err
		}
	}
	if o.mode.IsDir() {
		return fs.NewDir(remote, o.modTime), nil
	}
	return &o, nil
}

func (f *Fs) newEntry(remote string) (fs.DirEntry, error) {
	return f.newEntryWithFollow(remote, f.opt.FollowSymlinks)
}
func (f *Fs) newEntryFollowSymlinks(remote string) (fs.DirEntry, error) {
	return f.newEntryWithFollow(remote, true)
}
func (f *Fs) newEntryWithFollow(remote string, followSymlinks bool) (fs.DirEntry, error) {
	entry, err := f.device.Stat(path.Join(f.root, remote))
	if err != nil {
		if adb.HasErrCode(err, adb.FileNoExistError) {
			return nil, fs.ErrorObjectNotFound
		}
		return nil, errors.Wrapf(err, "Stat failed")
	}
	return f.entryForDirEntry(remote, entry, followSymlinks)
}

func (f *Fs) newObjectWithInfo(remote string, e *adb.DirEntry) Object {
	return Object{
		fs:      f,
		remote:  remote,
		size:    int64(e.Size),
		mode:    e.Mode,
		modTime: e.ModifiedAt,
	}
}

// NewObject finds the Object at remote.  If it can't be found
// it returns the error ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
	entry, err := f.newEntry(remote)
	if err != nil {
		return nil, err
	}
	obj, ok := entry.(fs.Object)
	if !ok {
		return nil, fs.ErrorObjectNotFound
	}
	return obj, nil
}

// Put in to the remote path with the modTime given of the given size
//
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
// return an error or upload it properly (rather than e.g. calling panic).
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	remote := src.Remote()
	// Temporary Object under construction - info filled in by Update()
	o := f.newObject(remote)
	err := o.Update(in, src, options...)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// newObject makes a half completed Object
func (f *Fs) newObject(remote string) *Object {
	return &Object{
		fs:     f,
		remote: remote,
	}
}

// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
func (f *Fs) Mkdir(dir string) error {
	p := path.Join(f.root, dir)
	output, code, err := f.device.execCommandWithExitCode("mkdir -p", p)
	switch err := err.(type) {
	case nil:
		return nil
	case adb.ShellExitError:
		entry, _ := f.newEntry(p)
		if _, ok := entry.(fs.Directory); ok {
			return nil
		}
		return errors.Errorf("mkdir %q failed with %d: %q", dir, code, output)
	default:
		return errors.Wrap(err, "mkdir")
	}
}

// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(dir string) error {
	p := path.Join(f.root, dir)
	output, code, err := f.device.execCommandWithExitCode("rmdir", p)
	switch err := err.(type) {
	case nil:
		return nil
	case adb.ShellExitError:
		return errors.Errorf("rmdir %q failed with %d: %q", dir, code, output)
	default:
		return errors.Wrap(err, "rmdir")
	}
}

// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
	return o.fs
}

// Return a string version
func (o *Object) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Remote returns the remote path
func (o *Object) Remote() string {
	return o.remote
}

// ModTime returns the modification date of the file
// It should return a best guess if one isn't available
func (o *Object) ModTime() time.Time {
	return o.modTime
}

// Size returns the size of the file
func (o *Object) Size() int64 {
	return o.size
}

// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *Object) Hash(hash.Type) (string, error) {
	return "", hash.ErrUnsupported
}

// Storable says whether this object can be stored
func (o *Object) Storable() bool {
	return true
}

// SetModTime sets the metadata on the object to set the modification date
func (o *Object) SetModTime(t time.Time) error {
	return o.fs.touchFunc(o, t)
}

func (o *Object) stat() error {
	return o.statStatArg(statArgC, path.Join(o.fs.root, o.remote))
}

func (o *Object) setMetadata(entry *adb.DirEntry) {
	// Don't overwrite the values if we don't need to
	// this avoids upsetting the race detector
	if o.size != int64(entry.Size) {
		o.size = int64(entry.Size)
	}
	if !o.modTime.Equal(entry.ModifiedAt) {
		o.modTime = entry.ModifiedAt
	}
	if o.mode != entry.Mode {
		o.mode = decodeEntryMode(uint32(entry.Mode))
	}
}

// Open opens the file for read.  Call Close() on the returned io.ReadCloser
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
	const blockSize = 1 << 12

	var offset, count int64 = 0, -1
	for _, option := range options {
		switch x := option.(type) {
		case *fs.RangeOption:
			offset, count = x.Decode(o.size)
		case *fs.SeekOption:
			offset = x.Offset
		default:
			if option.Mandatory() {
				fs.Logf(o, "Unsupported mandatory option: %v", option)
			}
		}
	}
	if offset > o.size {
		offset = o.size
	}
	if count < 0 {
		count = o.size - offset
	} else if count+offset > o.size {
		count = o.size - offset
	}
	fs.Debugf(o, "Open: remote: %q offset: %d count: %d", o.remote, offset, count)

	if count == 0 {
		return ioutil.NopCloser(bytes.NewReader(nil)), nil
	}
	offsetBlocks, offsetRest := offset/blockSize, offset%blockSize
	countBlocks := (count-1)/blockSize + 1

	conn, err := o.fs.device.execCommand(fmt.Sprintf("sh -c 'dd \"if=$0\" bs=%d skip=%d count=%d 2>/dev/null'", blockSize, offsetBlocks, countBlocks), path.Join(o.fs.root, o.remote))
	if err != nil {
		return nil, err
	}

	return &adbReader{
		ReadCloser: readers.NewLimitedReadCloser(conn, count+offsetRest),
		skip:       offsetRest,
		expected:   count,
	}, nil
}

// Update in to the object with the modTime given of the given size
//
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
// return an error or update the object properly (rather than e.g. calling panic).
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
	for _, option := range options {
		if option.Mandatory() {
			fs.Logf(option, "Unsupported mandatory option: %v", option)
		}
	}
	written, err := o.writeToFile(path.Join(o.fs.root, o.remote), in, 0666, src.ModTime())
	if err != nil {
		if removeErr := o.Remove(); removeErr != nil {
			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
		}
		return err
	}
	expected := src.Size()
	if expected == -1 {
		expected = written
	}
	for _, t := range []int64{100, 250, 500, 1000, 2500, 5000, 10000} {
		err = o.stat()
		if err != nil {
			return err
		}
		if o.size == expected {
			return nil
		}
		fs.Debugf(o, "Invalid size after update, expected: %d got: %d", expected, o.size)
		time.Sleep(time.Duration(t) * time.Millisecond)
	}
	return o.stat()
}

// Remove this object
func (o *Object) Remove() error {
	p := path.Join(o.fs.root, o.remote)
	output, code, err := o.fs.device.execCommandWithExitCode("rm", p)
	switch err := err.(type) {
	case nil:
		return nil
	case adb.ShellExitError:
		return errors.Errorf("rm %q failed with %d: %q", o.remote, code, output)
	default:
		return errors.Wrap(err, "rm")
	}
}

func (o *Object) writeToFile(path string, rd io.Reader, perms os.FileMode, modeTime time.Time) (written int64, err error) {
	dst, err := o.fs.device.OpenWrite(path, perms, modeTime)
	if err != nil {
		return
	}
	defer fs.CheckClose(dst, &err)
	return io.Copy(dst, rd)
}

type statFunc func(*Object) error

func (o *Object) statTry() error {
	o.fs.statFuncMu.Lock()
	defer o.fs.statFuncMu.Unlock()

	for _, f := range []statFunc{
		(*Object).statStatL, (*Object).statRealPath, (*Object).statReadLink,
	} {
		err := f(o)
		if err != nil {
			fs.Debugf(o, "%s", err)
		} else {
			o.fs.statFunc = f
			return nil
		}
	}

	return errors.Errorf("unable to resolve link target")
}

const (
	statArgLc = "-Lc"
	statArgC  = "-c"
)

func (o *Object) statStatL() error {
	return o.statStatArg(statArgLc, path.Join(o.fs.root, o.remote))
}

func (o *Object) statStatArg(arg, path string) error {
	output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("stat %s %s", arg, "%f,%s,%Y"), path)
	output = strings.TrimSpace(output)
	switch err := err.(type) {
	case nil:
	case adb.ShellExitError:
		return errors.Errorf("stat %q failed with %d: %q", o.remote, code, output)
	default:
		return errors.Wrap(err, "stat")
	}

	parts := strings.Split(output, ",")
	if len(parts) != 3 {
		return errors.Errorf("stat %q invalid output %q", o.remote, output)
	}

	mode, err := strconv.ParseUint(parts[0], 16, 32)
	if err != nil {
		return errors.Errorf("stat %q invalid output %q", o.remote, output)
	}
	size, err := strconv.ParseUint(parts[1], 10, 64)
	if err != nil {
		return errors.Errorf("stat %q invalid output %q", o.remote, output)
	}
	modTime, err := strconv.ParseInt(parts[2], 10, 64)
	if err != nil {
		return errors.Errorf("stat %q invalid output %q", o.remote, output)
	}

	o.size = int64(size)
	o.modTime = time.Unix(modTime, 0)
	o.mode = decodeEntryMode(uint32(mode))
	return nil
}

func (o *Object) statReadLink() error {
	p := path.Join(o.fs.root, o.remote)
	output, code, err := o.fs.device.execCommandWithExitCode("readlink -f", p)
	output = strings.TrimSuffix(output, "\n")
	switch err := err.(type) {
	case nil:
	case adb.ShellExitError:
		return errors.Errorf("readlink %q failed with %d: %q", o.remote, code, output)
	default:
		return errors.Wrap(err, "readlink")
	}
	return o.statStatArg(statArgC, output)
}
func (o *Object) statRealPath() error {
	p := path.Join(o.fs.root, o.remote)
	output, code, err := o.fs.device.execCommandWithExitCode("realpath", p)
	output = strings.TrimSuffix(output, "\n")
	switch err := err.(type) {
	case nil:
	case adb.ShellExitError:
		return errors.Errorf("realpath %q failed with %d: %q", o.remote, code, output)
	default:
		return errors.Wrap(err, "realpath")
	}
	return o.statStatArg(statArgC, output)
}

type touchFunc func(*Object, time.Time) error

func (o *Object) touchTry(t time.Time) error {
	o.fs.touchFuncMu.Lock()
	defer o.fs.touchFuncMu.Unlock()

	for _, f := range []touchFunc{
		(*Object).touchCmd, (*Object).touchCd,
	} {
		err := f(o, t)
		if err != nil {
			fs.Debugf(o, "%s", err)
		} else {
			o.fs.touchFunc = f
			return nil
		}
	}

	return errors.Errorf("unable to resolve link target")
}

const (
	touchArgCmd = "-cmd"
	touchArgCd  = "-cd"
)

func (o *Object) touchCmd(t time.Time) error {
	return o.touchStatArg(touchArgCmd, path.Join(o.fs.root, o.remote), t)
}
func (o *Object) touchCd(t time.Time) error {
	return o.touchStatArg(touchArgCd, path.Join(o.fs.root, o.remote), t)
}

func (o *Object) touchStatArg(arg, path string, t time.Time) error {
	output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("touch %s %s", arg, t.Format(time.RFC3339Nano)), path)
	output = strings.TrimSpace(output)
	switch err := err.(type) {
	case nil:
	case adb.ShellExitError:
		return errors.Errorf("touch %q failed with %d: %q", o.remote, code, output)
	default:
		return errors.Wrap(err, "touch")
	}

	err = o.stat()
	if err != nil {
		return err
	}
	if diff, ok := checkTimeEqualWithPrecision(t, o.modTime, o.fs.Precision()); !ok {
		return errors.Errorf("touch %q to %s was ineffective: %d", o.remote, t, diff)
	}

	return nil
}

func checkTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
	dt := t0.Sub(t1)
	if dt >= precision || dt <= -precision {
		return dt, false
	}
	return dt, true
}

func decodeEntryMode(entryMode uint32) os.FileMode {
	const (
		unixIFBLK  = 0x6000
		unixIFMT   = 0xf000
		unixIFCHR  = 0x2000
		unixIFDIR  = 0x4000
		unixIFIFO  = 0x1000
		unixIFLNK  = 0xa000
		unixIFREG  = 0x8000
		unixIFSOCK = 0xc000
		unixISGID  = 0x400
		unixISUID  = 0x800
		unixISVTX  = 0x200
	)

	mode := os.FileMode(entryMode & 0777)
	switch entryMode & unixIFMT {
	case unixIFBLK:
		mode |= os.ModeDevice
	case unixIFCHR:
		mode |= os.ModeDevice | os.ModeCharDevice
	case unixIFDIR:
		mode |= os.ModeDir
	case unixIFIFO:
		mode |= os.ModeNamedPipe
	case unixIFLNK:
		mode |= os.ModeSymlink
	case unixIFREG:
		// nothing to do
	case unixIFSOCK:
		mode |= os.ModeSocket
	}
	if entryMode&unixISGID != 0 {
		mode |= os.ModeSetgid
	}
	if entryMode&unixISUID != 0 {
		mode |= os.ModeSetuid
	}
	if entryMode&unixISVTX != 0 {
		mode |= os.ModeSticky
	}
	return mode
}

type execDevice struct {
	*adb.Device
}

func (d *execDevice) execCommandWithExitCode(cmd string, arg string) (string, int, error) {
	cmdLine := fmt.Sprintf("sh -c '%s \"$0\"; echo :$?' '%s'", cmd, strings.Replace(arg, "'", "'\\''", -1))
	fs.Debugf("adb", "exec: %s", cmdLine)
	conn, err := d.execCommand(cmdLine)
	if err != nil {
		return "", -1, err
	}

	resp, err := conn.ReadUntilEof()
	if err != nil {
		return "", -1, errors.Wrap(err, "ExecCommand")
	}

	outStr := string(resp)
	idx := strings.LastIndexByte(outStr, ':')
	if idx == -1 {
		return outStr, -1, fmt.Errorf("adb shell aborted, can not parse exit code")
	}
	exitCode, _ := strconv.Atoi(strings.TrimSpace(outStr[idx+1:]))
	if exitCode != 0 {
		err = adb.ShellExitError{Command: cmdLine, ExitCode: exitCode}
	}
	return outStr[:idx], exitCode, err
}

func (d *execDevice) execCommand(cmd string, args ...string) (*wire.Conn, error) {
	cmd = prepareCommandLineEscaped(cmd, args...)
	conn, err := d.Dial()
	if err != nil {
		return nil, errors.Wrap(err, "ExecCommand")
	}
	defer func() {
		if err != nil && conn != nil {
			_ = conn.Close()
		}
	}()

	req := fmt.Sprintf("exec:%s", cmd)

	if err = conn.SendMessage([]byte(req)); err != nil {
		return nil, errors.Wrap(err, "ExecCommand")
	}
	if _, err = conn.ReadStatus(req); err != nil {
		return nil, errors.Wrap(err, "ExecCommand")
	}
	return conn, nil
}

func prepareCommandLineEscaped(cmd string, args ...string) string {
	for i, arg := range args {
		args[i] = fmt.Sprintf("'%s'", strings.Replace(arg, "'", "'\\''", -1))
	}

	// Prepend the command to the args array.
	if len(args) > 0 {
		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
	}

	return cmd
}

type adbReader struct {
	io.ReadCloser
	skip     int64
	read     int64
	expected int64
}

func (r *adbReader) Read(b []byte) (n int, err error) {
	n, err = r.ReadCloser.Read(b)
	if s := r.skip; n > 0 && s > 0 {
		_n := int64(n)
		if _n <= s {
			r.skip -= _n
			return r.Read(b)
		}
		r.skip = 0
		copy(b, b[s:n])
		n -= int(s)
	}
	r.read += int64(n)
	if err == io.EOF && r.read < r.expected {
		fs.Debugf("adb", "Read: read: %d expected: %d n: %d", r.read, r.expected, n)
		return n, io.ErrUnexpectedEOF
	}
	return n, err
}