Add storage

storage/storage.go

@@ -0,0 +1,231 @@
+package storage
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path"
+
+	"github.com/fd0/khepri/hashing"
+)
+
+const (
+	dirMode    = 0700
+	objectPath = "objects"
+	refPath    = "refs"
+	tempPath   = "tmp"
+)
+
+type Repository interface {
+	Put(reader io.Reader) (ID, error)
+	PutFile(path string) (ID, error)
+	Get(ID) (io.Reader, error)
+	Test(ID) (bool, error)
+	Link(name string, id ID) error
+	Unlink(name string) error
+	Resolve(name string) (ID, error)
+}
+
+var (
+	ErrIDDoesNotExist = errors.New("ID does not exist")
+)
+
+// References content within a repository.
+type ID []byte
+
+func (id ID) String() string {
+	return hex.EncodeToString(id)
+}
+
+// Equal compares an ID to another other.
+func (id ID) Equal(other ID) bool {
+	return bytes.Equal(id, other)
+}
+
+// EqualString compares this ID to another one, given as a string.
+func (id ID) EqualString(other string) (bool, error) {
+	s, err := hex.DecodeString(other)
+	if err != nil {
+		return false, err
+	}
+
+	return id.Equal(ID(s)), nil
+}
+
+// Name stands for the alias given to an ID.
+type Name string
+
+func (n Name) Encode() string {
+	return url.QueryEscape(string(n))
+}
+
+type Dir struct {
+	path string
+	hash func() hash.Hash
+}
+
+// NewDir creates a new dir-baked repository at the given path.
+func NewDir(path string) (*Dir, error) {
+	d := &Dir{
+		path: path,
+		hash: sha256.New,
+	}
+
+	err := d.create()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return d, nil
+}
+
+func (r *Dir) create() error {
+	dirs := []string{
+		r.path,
+		path.Join(r.path, objectPath),
+		path.Join(r.path, refPath),
+		path.Join(r.path, tempPath),
+	}
+
+	for _, dir := range dirs {
+		err := os.MkdirAll(dir, dirMode)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// SetHash changes the hash function used for deriving IDs. Default is SHA256.
+func (r *Dir) SetHash(h func() hash.Hash) {
+	r.hash = h
+}
+
+// Path returns the directory used for this repository.
+func (r *Dir) Path() string {
+	return r.path
+}
+
+// Put saves content and returns the ID.
+func (r *Dir) Put(reader io.Reader) (ID, error) {
+	// save contents to tempfile, hash while writing
+	file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
+	if err != nil {
+		return nil, err
+	}
+
+	rd := hashing.NewReader(reader, r.hash)
+	_, err = io.Copy(file, rd)
+	if err != nil {
+		return nil, err
+	}
+
+	err = file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// move file to final name using hash of contents
+	id := ID(rd.Hash())
+	filename := path.Join(r.path, objectPath, id.String())
+	err = os.Rename(file.Name(), filename)
+	if err != nil {
+		return nil, err
+	}
+
+	return id, nil
+}
+
+// PutFile saves a file's content to the repository and returns the ID.
+func (r *Dir) PutFile(path string) (ID, error) {
+	f, err := os.Open(path)
+	defer f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return r.Put(f)
+}
+
+// Test returns true if the given ID exists in the repository.
+func (r *Dir) Test(id ID) (bool, error) {
+	// try to open file
+	file, err := os.Open(path.Join(r.path, objectPath, id.String()))
+	defer func() {
+		file.Close()
+	}()
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}
+
+// Get returns a reader for the content stored under the given ID.
+func (r *Dir) Get(id ID) (io.Reader, error) {
+	// try to open file
+	file, err := os.Open(path.Join(r.path, objectPath, id.String()))
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}
+
+// Unlink removes a named ID.
+func (r *Dir) Unlink(name string) error {
+	return os.Remove(path.Join(r.path, refPath, Name(name).Encode()))
+}
+
+// Link assigns a name to an ID. Name must be unique in this repository and ID must exist.
+func (r *Dir) Link(name string, id ID) error {
+	exist, err := r.Test(id)
+	if err != nil {
+		return err
+	}
+
+	if !exist {
+		return ErrIDDoesNotExist
+	}
+
+	// create file, write id
+	f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode()))
+	defer f.Close()
+
+	if err != nil {
+		return err
+	}
+
+	f.Write(id)
+	return nil
+}
+
+// Resolve returns the ID associated with the given name.
+func (r *Dir) Resolve(name string) (ID, error) {
+	f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode()))
+	defer f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	id := make([]byte, r.hash().Size())
+	_, err = io.ReadFull(f, id)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return ID(id), nil
+}

storage/storage_suite_test.go

@@ -0,0 +1,13 @@
+package storage_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestStorage(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Storage Suite")
+}

storage/storage_test.go

@@ -0,0 +1,98 @@
+package storage_test
+
+import (
+	"bytes"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/fd0/khepri/storage"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var TestStrings = []struct {
+	id   string
+	data string
+}{
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+}
+
+var _ = Describe("Storage", func() {
+	var (
+		tempdir string
+		repo    storage.Repository
+		err     error
+		id      storage.ID
+	)
+
+	BeforeEach(func() {
+		tempdir, err = ioutil.TempDir("", "khepri-test-")
+		if err != nil {
+			panic(err)
+		}
+		repo, err = storage.NewDir(tempdir)
+		if err != nil {
+			panic(err)
+		}
+	})
+
+	AfterEach(func() {
+		err = os.RemoveAll(tempdir)
+		if err != nil {
+			panic(err)
+		}
+		// fmt.Fprintf(os.Stderr, "leaving tempdir %s", tempdir)
+		tempdir = ""
+	})
+
+	Describe("Repository", func() {
+		Context("File Operations", func() {
+			It("Should detect non-existing file", func() {
+				for _, test := range TestStrings {
+					id, err := hex.DecodeString(test.id)
+					Expect(err).NotTo(HaveOccurred())
+
+					// try to get string out, should fail
+					ret, err := repo.Test(id)
+					Expect(ret).Should(Equal(false))
+				}
+			})
+
+			It("Should Add File", func() {
+				for _, test := range TestStrings {
+					// store string in repository
+					id, err = repo.Put(strings.NewReader(test.data))
+
+					Expect(err).NotTo(HaveOccurred())
+					Expect(id.String()).Should(Equal(test.id))
+
+					// try to get it out again
+					var buf bytes.Buffer
+					rd, err := repo.Get(id)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(rd).ShouldNot(BeNil())
+
+					// compare content
+					Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data))))
+					Expect(buf.Bytes()).Should(Equal([]byte(test.data)))
+
+					// store id under name
+					err = repo.Link(test.data, id)
+					Expect(err).NotTo(HaveOccurred())
+
+					// resolve again
+					Expect(repo.Resolve(test.data)).Should(Equal(id))
+
+					// remove link
+					Expect(repo.Unlink(test.data)).NotTo(HaveOccurred())
+				}
+			})
+		})
+	})
+})