First test implementation

storage/id.go

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+)
+
+// References content within a repository.
+type ID []byte
+
+// ParseID converts the given string to an ID.
+func ParseID(s string) (ID, error) {
+	b, err := hex.DecodeString(s)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return ID(b), nil
+}
+func (id ID) String() string {
+	return hex.EncodeToString(id)
+}
+
+// Equal compares an ID to another other.
+func (id ID) Equal(other ID) bool {
+	return bytes.Equal(id, other)
+}
+
+// EqualString compares this ID to another one, given as a string.
+func (id ID) EqualString(other string) (bool, error) {
+	s, err := hex.DecodeString(other)
+	if err != nil {
+		return false, err
+	}
+
+	return id.Equal(ID(s)), nil
+}
+
+func (id ID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(id.String())
+}
+
+func (id *ID) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	*id = make([]byte, len(s)/2)
+	_, err = hex.Decode(*id, []byte(s))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

storage/repository.go

@@ -1,7 +1,6 @@
 package storage
 
 import (
-	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
@@ -25,6 +24,7 @@ const (
 type Repository interface {
 	Put(reader io.Reader) (ID, error)
 	PutFile(path string) (ID, error)
+	PutRaw([]byte) (ID, error)
 	Get(ID) (io.Reader, error)
 	Test(ID) (bool, error)
 	Remove(ID) error
@@ -37,28 +37,6 @@ var (
 	ErrIDDoesNotExist = errors.New("ID does not exist")
 )
 
-// References content within a repository.
-type ID []byte
-
-func (id ID) String() string {
-	return hex.EncodeToString(id)
-}
-
-// Equal compares an ID to another other.
-func (id ID) Equal(other ID) bool {
-	return bytes.Equal(id, other)
-}
-
-// EqualString compares this ID to another one, given as a string.
-func (id ID) EqualString(other string) (bool, error) {
-	s, err := hex.DecodeString(other)
-	if err != nil {
-		return false, err
-	}
-
-	return id.Equal(ID(s)), nil
-}
-
 // Name stands for the alias given to an ID.
 type Name string
 
@@ -156,6 +134,40 @@ func (r *DirRepository) PutFile(path string) (ID, error) {
 	return r.Put(f)
 }
 
+// PutRaw saves a []byte's content to the repository and returns the ID.
+func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
+	// save contents to tempfile, hash while writing
+	file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
+	if err != nil {
+		return nil, err
+	}
+
+	wr := hashing.NewWriter(file, r.hash)
+	n, err := wr.Write(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	if n != len(buf) {
+		return nil, errors.New("not all bytes written")
+	}
+
+	err = file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// move file to final name using hash of contents
+	id := ID(wr.Hash())
+	filename := path.Join(r.path, objectPath, id.String())
+	err = os.Rename(file.Name(), filename)
+	if err != nil {
+		return nil, err
+	}
+
+	return id, nil
+}
+
 // Test returns true if the given ID exists in the repository.
 func (r *DirRepository) Test(id ID) (bool, error) {
 	// try to open file

storage/repository_test.go

@@ -2,7 +2,6 @@ package storage_test
 
 import (
 	"bytes"
-	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"os"
@@ -55,7 +54,7 @@ var _ = Describe("Storage", func() {
 		Context("File Operations", func() {
 			It("Should detect non-existing file", func() {
 				for _, test := range TestStrings {
-					id, err := hex.DecodeString(test.id)
+					id, err := storage.ParseID(test.id)
 					Expect(err).NotTo(HaveOccurred())
 
 					// try to get string out, should fail
@@ -96,6 +95,15 @@ var _ = Describe("Storage", func() {
 					Expect(repo.Remove(id))
 				}
 			})
+
+			It("Should Add Buffer", func() {
+				for _, test := range TestStrings {
+					// store buf in repository
+					id, err := repo.PutRaw([]byte(test.data))
+					Expect(err).NotTo(HaveOccurred())
+					Expect(id.String()).To(Equal(test.id))
+				}
+			})
 		})
 	})
 })

storage/tree.go

@@ -0,0 +1,53 @@
+package storage
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"syscall"
+	"time"
+)
+
+type Tree struct {
+	Nodes []Node `json:"nodes"`
+}
+
+type Node struct {
+	Name    string      `json:"name"`
+	Mode    os.FileMode `json:"mode"`
+	ModTime time.Time   `json:"mtime"`
+	User    uint32      `json:"user"`
+	Group   uint32      `json:"group"`
+	Content ID          `json:"content"`
+}
+
+func NewTree() *Tree {
+	return &Tree{
+		Nodes: []Node{},
+	}
+}
+
+func (t *Tree) Restore(r io.Reader) error {
+	dec := json.NewDecoder(r)
+	return dec.Decode(t)
+}
+
+func (t *Tree) Save(w io.Writer) error {
+	enc := json.NewEncoder(w)
+	return enc.Encode(t)
+}
+
+func NodeFromFileInfo(fi os.FileInfo) Node {
+	node := Node{
+		Name:    fi.Name(),
+		Mode:    fi.Mode(),
+		ModTime: fi.ModTime(),
+	}
+
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+		node.User = stat.Uid
+		node.Group = stat.Gid
+	}
+
+	return node
+}

storage/tree_test.go

@@ -0,0 +1,77 @@
+package storage_test
+
+import (
+	"bytes"
+	"strings"
+	"time"
+
+	"github.com/fd0/khepri/storage"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func parseTime(str string) time.Time {
+	t, err := time.Parse(time.RFC3339Nano, str)
+	if err != nil {
+		panic(err)
+	}
+
+	return t
+}
+
+var _ = Describe("Tree", func() {
+	var t *storage.Tree
+	var raw string
+
+	BeforeEach(func() {
+		t = new(storage.Tree)
+		t.Nodes = []storage.Node{
+			storage.Node{
+				Name:    "foobar",
+				Mode:    0755,
+				ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"),
+				User:    1000,
+				Group:   1001,
+				Content: []byte{0x41, 0x42, 0x43},
+			},
+			storage.Node{
+				Name:    "baz",
+				Mode:    0755,
+				User:    1000,
+				ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"),
+				Group:   1001,
+				Content: []byte("\xde\xad\xbe\xef\xba\xdc\x0d\xe0"),
+			},
+		}
+
+		raw = `{"nodes":[{"name":"foobar","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","user":1000,"group":1001,"content":"414243"},{"name":"baz","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","user":1000,"group":1001,"content":"deadbeefbadc0de0"}]}`
+	})
+
+	It("Should save", func() {
+		var buf bytes.Buffer
+		t.Save(&buf)
+		Expect(strings.TrimRight(buf.String(), "\n")).To(Equal(raw))
+
+		t2 := new(storage.Tree)
+		err := t2.Restore(&buf)
+		Expect(err).NotTo(HaveOccurred())
+
+		// test tree for equality
+		Expect(t2).To(Equal(t))
+
+		// test nodes for equality
+		for i, n := range t.Nodes {
+			Expect(n.Content).To(Equal(t2.Nodes[i].Content))
+		}
+	})
+
+	It("Should restore", func() {
+		buf := bytes.NewBufferString(raw)
+		t2 := new(storage.Tree)
+		err := t2.Restore(buf)
+		Expect(err).NotTo(HaveOccurred())
+
+		// test if tree has correctly been restored
+		Expect(t2).To(Equal(t))
+	})
+})

storagetest/main.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"io"
 	"log"
 	"os"
+	"path/filepath"
 
 	"github.com/fd0/khepri/storage"
 )
@@ -20,31 +22,71 @@ func hash(filename string) (storage.ID, error) {
 	return h.Sum([]byte{}), nil
 }
 
-func archive_dir(repo storage.Repository, path string) {
+func archive_dir(repo storage.Repository, path string) (storage.ID, error) {
 	log.Printf("archiving dir %q", path)
-	// items := make()
-	// filepath.Walk(path, func(item string, info os.FileInfo, err error) error {
-	// 	log.Printf("  archiving %q", item)
+	dir, err := os.Open(path)
+	if err != nil {
+		log.Printf("open(%q): %v\n", path, err)
+		return nil, err
+	}
 
-	// 	if item != path && info.IsDir() {
-	// 		archive_dir(repo, item)
-	// 	} else {
+	entries, err := dir.Readdir(-1)
+	if err != nil {
+		log.Printf("readdir(%q): %v\n", path, err)
+		return nil, err
+	}
 
-	// 	}
+	t := storage.NewTree()
+	for _, e := range entries {
+		node := storage.NodeFromFileInfo(e)
 
-	// 	return nil
-	// })
+		var id storage.ID
+		var err error
+
+		if e.IsDir() {
+			id, err = archive_dir(repo, filepath.Join(path, e.Name()))
+		} else {
+			id, err = repo.PutFile(filepath.Join(path, e.Name()))
+		}
+
+		node.Content = id
+
+		t.Nodes = append(t.Nodes, node)
+
+		if err != nil {
+			log.Printf("  error storing %q: %v\n", e.Name(), err)
+			continue
+		}
+	}
+
+	log.Printf("  dir %q: %v entries", path, len(t.Nodes))
+
+	var buf bytes.Buffer
+	t.Save(&buf)
+	id, err := repo.PutRaw(buf.Bytes())
+
+	if err != nil {
+		log.Printf("error saving tree to repo: %v", err)
+	}
+
+	log.Printf("tree for %q saved at %s", path, id)
+
+	return id, nil
 }
 
 func main() {
-	repo, err := storage.NewDir("repo")
+	if len(os.Args) <= 2 {
+		log.Fatalf("usage: %s repo [add|link|putdir] ...", os.Args[0])
+	}
+
+	repo, err := storage.NewDirRepository(os.Args[1])
 	if err != nil {
 		log.Fatalf("error: %v", err)
 	}
 
-	switch os.Args[1] {
+	switch os.Args[2] {
 	case "add":
-		for _, file := range os.Args[2:] {
+		for _, file := range os.Args[3:] {
 			f, err := os.Open(file)
 			if err != nil {
 				log.Printf("error opening file %q: %v", file, err)
@@ -59,8 +101,8 @@ func main() {
 			log.Printf("archived file %q as ID %v", file, id)
 		}
 	case "link":
-		file := os.Args[2]
-		name := os.Args[3]
+		file := os.Args[3]
+		name := os.Args[4]
 
 		id, err := hash(file)
 		if err != nil {
@@ -85,10 +127,10 @@ func main() {
 			log.Fatalf("error linking name %q to id %v", name, id)
 		}
 	case "putdir":
-		for _, dir := range os.Args[2:] {
+		for _, dir := range os.Args[3:] {
 			archive_dir(repo, dir)
 		}
 	default:
-		log.Fatalf("unknown command: %q", os.Args[1])
+		log.Fatalf("unknown command: %q", os.Args[2])
 	}
 }