Add SFTP backend

backend/interface.go

@@ -22,5 +22,7 @@ type Server interface {
 	Remove(Type, ID) error
 	Version() uint
 
+	Close() error
+
 	Location() string
 }

backend/local.go

@@ -186,7 +186,12 @@ func (b *Local) Create(t Type, data []byte) (ID, error) {
 		return nil, err
 	}
 
-	// close tempfile, return id
+	err = file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// return id
 	id := IDFromData(data)
 	err = b.renameFile(file, t, id)
 	if err != nil {
@@ -276,3 +281,8 @@ func (b *Local) List(t Type) (IDs, error) {
 func (b *Local) Version() uint {
 	return b.ver
 }
+
+// Close closes the repository
+func (b *Local) Close() error {
+	return nil
+}

backend/sftp.go

@@ -0,0 +1,369 @@
+package backend
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/sftp"
+)
+
+const (
+	// dirMode         = 0700
+	// blobPath        = "blobs"
+	// snapshotPath    = "snapshots"
+	// treePath        = "trees"
+	// lockPath        = "locks"
+	// keyPath         = "keys"
+	// tempPath        = "tmp"
+	// versionFileName = "version"
+
+	tempfileRandomSuffixLength = 10
+)
+
+type SFTP struct {
+	c   *sftp.Client
+	p   string
+	ver uint
+
+	cmd *exec.Cmd
+}
+
+func start_client(program string, args ...string) (*SFTP, error) {
+	// Connect to a remote host and request the sftp subsystem via the 'ssh'
+	// command.  This assumes that passwordless login is correctly configured.
+	cmd := exec.Command(program, args...)
+
+	// send errors from ssh to stderr
+	cmd.Stderr = os.Stderr
+
+	// get stdin and stdout
+	wr, err := cmd.StdinPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+	rd, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// start the process
+	if err := cmd.Start(); err != nil {
+		log.Fatal(err)
+	}
+
+	// open the SFTP session
+	client, err := sftp.NewClientPipe(rd, wr)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return &SFTP{c: client, cmd: cmd}, nil
+}
+
+// OpenSFTP opens an sftp backend. When the command is started via
+// exec.Command, it is expected to speak sftp on stdin/stdout. The repository
+// is expected at the given path.
+func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) {
+	sftp, err := start_client(program, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	// test if all necessary dirs and files are there
+	items := []string{
+		dir,
+		filepath.Join(dir, blobPath),
+		filepath.Join(dir, snapshotPath),
+		filepath.Join(dir, treePath),
+		filepath.Join(dir, lockPath),
+		filepath.Join(dir, keyPath),
+		filepath.Join(dir, tempPath),
+	}
+	for _, d := range items {
+		if _, err := sftp.c.Lstat(d); err != nil {
+			return nil, fmt.Errorf("%s does not exist", d)
+		}
+	}
+
+	// read version file
+	f, err := sftp.c.Open(filepath.Join(dir, versionFileName))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read version file: %v\n", err)
+	}
+
+	buf := make([]byte, 100)
+	n, err := f.Read(buf)
+	if err != nil && err != io.EOF {
+		return nil, err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	version, err := strconv.Atoi(strings.TrimSpace(string(buf[:n])))
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert version to integer: %v\n", err)
+	}
+
+	if version != BackendVersion {
+		return nil, fmt.Errorf("wrong version %d", version)
+	}
+
+	// check version
+	if version != BackendVersion {
+		return nil, fmt.Errorf("wrong version %d", version)
+	}
+
+	sftp.p = dir
+
+	return sftp, nil
+}
+
+// CreateSFTP creates all the necessary files and directories for a new sftp
+// backend at dir.
+func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
+	sftp, err := start_client(program, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	versionFile := filepath.Join(dir, versionFileName)
+	dirs := []string{
+		dir,
+		filepath.Join(dir, blobPath),
+		filepath.Join(dir, snapshotPath),
+		filepath.Join(dir, treePath),
+		filepath.Join(dir, lockPath),
+		filepath.Join(dir, keyPath),
+		filepath.Join(dir, tempPath),
+	}
+
+	// test if version file already exists
+	_, err = sftp.c.Lstat(versionFile)
+	if err == nil {
+		return nil, errors.New("version file already exists")
+	}
+
+	// test if directories already exist
+	for _, d := range dirs[1:] {
+		if _, err := sftp.c.Lstat(d); err == nil {
+			return nil, fmt.Errorf("dir %s already exists", d)
+		}
+	}
+
+	// create paths for blobs, refs and temp
+	for _, d := range dirs {
+		// TODO: implement client.MkdirAll() and set mode to dirMode
+		_, err = sftp.c.Lstat(d)
+		if err != nil {
+			err = sftp.c.Mkdir(d)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// create version file
+	f, err := sftp.c.Create(versionFile)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = f.Write([]byte(strconv.Itoa(BackendVersion)))
+	if err != nil {
+		return nil, err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	err = sftp.c.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	err = sftp.cmd.Wait()
+	if err != nil {
+		return nil, err
+	}
+
+	// open repository
+	return OpenSFTP(dir, program, args...)
+}
+
+// Location returns this backend's location (the directory name).
+func (r *SFTP) Location() string {
+	return r.p
+}
+
+// Return temp directory in correct directory for this backend.
+func (r *SFTP) tempFile() (string, *sftp.File, error) {
+	// choose random suffix
+	buf := make([]byte, tempfileRandomSuffixLength)
+	n, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if n != len(buf) {
+		return "", nil, errors.New("unable to generate enough random bytes for temp file")
+	}
+
+	// construct tempfile name
+	name := filepath.Join(r.p, tempPath, fmt.Sprintf("temp-%s", hex.EncodeToString(buf)))
+
+	// create file in temp dir
+	f, err := r.c.Create(name)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return name, f, nil
+}
+
+// Rename temp file to final name according to type and ID.
+func (r *SFTP) renameFile(filename string, t Type, id ID) error {
+	return r.c.Rename(filename, filepath.Join(r.dir(t), id.String()))
+}
+
+// Construct directory for given Type.
+func (r *SFTP) dir(t Type) string {
+	var n string
+	switch t {
+	case Blob:
+		n = blobPath
+	case Snapshot:
+		n = snapshotPath
+	case Tree:
+		n = treePath
+	case Lock:
+		n = lockPath
+	case Key:
+		n = keyPath
+	}
+	return filepath.Join(r.p, n)
+}
+
+// Create stores new content of type t and data and returns the ID.
+func (r *SFTP) Create(t Type, data []byte) (ID, error) {
+	// TODO: make sure that tempfile is removed upon error
+
+	// create tempfile in repository
+	var err error
+	filename, file, err := r.tempFile()
+	if err != nil {
+		return nil, err
+	}
+
+	// write data to tempfile
+	_, err = file.Write(data)
+	if err != nil {
+		return nil, err
+	}
+
+	err = file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// return id
+	id := IDFromData(data)
+	err = r.renameFile(filename, t, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return id, nil
+}
+
+// Construct path for given Type and ID.
+func (r *SFTP) filename(t Type, id ID) string {
+	return filepath.Join(r.dir(t), id.String())
+}
+
+// Get returns the content stored under the given ID.
+func (r *SFTP) Get(t Type, id ID) ([]byte, error) {
+	// try to open file
+	file, err := r.c.Open(r.filename(t, id))
+	defer file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// read all
+	buf, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+// Test returns true if a blob of the given type and ID exists in the backend.
+func (r *SFTP) Test(t Type, id ID) (bool, error) {
+	// try to open file
+	file, err := r.c.Open(r.filename(t, id))
+	defer func() {
+		file.Close()
+	}()
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}
+
+// Remove removes the content stored at ID.
+func (r *SFTP) Remove(t Type, id ID) error {
+	return r.c.Remove(r.filename(t, id))
+}
+
+// List lists all objects of a given type.
+func (r *SFTP) List(t Type) (IDs, error) {
+	list, err := r.c.ReadDir(r.dir(t))
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make(IDs, 0, len(list))
+	for _, item := range list {
+		id, err := ParseID(item.Name())
+		// ignore everything that does not parse as an ID
+		if err != nil {
+			continue
+		}
+		ids = append(ids, id)
+	}
+
+	return ids, nil
+}
+
+// Version returns the version of this local backend.
+func (r *SFTP) Version() uint {
+	return r.ver
+}
+
+// Close closes the sftp connection and terminates the underlying command.
+func (s *SFTP) Close() error {
+	s.c.Close()
+	// TODO: add timeout after which the process is killed
+	return s.cmd.Wait()
+}

cmd/khepri/cmd_backup.go

@@ -21,7 +21,7 @@ func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
 		fmt.Fprintf(os.Stderr, "err: %v\n", err)
 	}
 	arch.Error = func(dir string, fi os.FileInfo, err error) error {
-		fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi)
+		fmt.Fprintf(os.Stderr, "error for %s: %v\n%v\n", dir, err, fi)
 		return err
 	}
 

cmd/khepri/cmd_init.go

@@ -1,34 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/fd0/khepri"
-	"github.com/fd0/khepri/backend"
-)
-
-func commandInit(path string) error {
-	pw := read_password("enter password for new backend: ")
-	pw2 := read_password("enter password again: ")
-
-	if pw != pw2 {
-		errx(1, "passwords do not match")
-	}
-
-	be, err := backend.CreateLocal(path)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "creating local backend at %s failed: %v\n", path, err)
-		os.Exit(1)
-	}
-
-	_, err = khepri.CreateKey(be, pw)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "creating key in local backend at %s failed: %v\n", path, err)
-		os.Exit(1)
-	}
-
-	fmt.Printf("created khepri backend at %s\n", be.Location())
-
-	return nil
-}

cmd/khepri/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"sort"
 	"strings"
@@ -46,6 +47,77 @@ func read_password(prompt string) string {
 	return string(pw)
 }
 
+func commandInit(repo string) error {
+	pw := read_password("enter password for new backend: ")
+	pw2 := read_password("enter password again: ")
+
+	if pw != pw2 {
+		errx(1, "passwords do not match")
+	}
+
+	be, err := create(repo)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", repo, err)
+		os.Exit(1)
+	}
+
+	_, err = khepri.CreateKey(be, pw)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", repo, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("created khepri backend at %s\n", be.Location())
+
+	return nil
+}
+
+// Open the backend specified by URI.
+// Valid formats are:
+// * /foo/bar -> local repository at /foo/bar
+// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
+// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
+func open(u string) (backend.Server, error) {
+	url, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	if url.Scheme == "" {
+		return backend.OpenLocal(url.Path)
+	} else {
+		args := []string{url.Host}
+		if url.User != nil && url.User.Username() != "" {
+			args = append(args, "-l")
+			args = append(args, url.User.Username())
+		}
+		args = append(args, "-s")
+		args = append(args, "sftp")
+		return backend.OpenSFTP(url.Path[1:], "ssh", args...)
+	}
+}
+
+// Create the backend specified by URI.
+func create(u string) (backend.Server, error) {
+	url, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	if url.Scheme == "" {
+		return backend.CreateLocal(url.Path)
+	} else {
+		args := []string{url.Host}
+		if url.User != nil && url.User.Username() != "" {
+			args = append(args, "-l")
+			args = append(args, url.User.Username())
+		}
+		args = append(args, "-s")
+		args = append(args, "sftp")
+		return backend.CreateSFTP(url.Path[1:], "ssh", args...)
+	}
+}
+
 func init() {
 	commands = make(map[string]commandFunc)
 	commands["backup"] = commandBackup
@@ -94,7 +166,7 @@ func main() {
 	}
 
 	// read_password("enter password: ")
-	repo, err := backend.OpenLocal(Opts.Repo)
+	repo, err := open(Opts.Repo)
 	if err != nil {
 		errx(1, "unable to open repo: %v", err)
 	}