KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Introduce Type for DirRepository objects

Browse source
This commit is contained in:
Alexander Neumann 2014-07-30 23:11:23 +02:00
parent 13bb557cdc
commit a71b49ebb9
2 changed files with 170 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

191
repository.go
View file

@ -2,21 +2,22 @@ package khepri
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
)
const (
dirMode = 0700
objectPath = "objects"
refPath = "refs"
tempPath = "tmp"
dirMode = 0700
blobPath = "blobs"
refPath = "refs"
tempPath = "tmp"
)
var (
@ -35,6 +36,36 @@ type DirRepository struct {
hash func() hash.Hash
}
type Type int
const (
TypeUnknown = iota
TypeBlob
TypeRef
)
func NewTypeFromString(s string) Type {
switch s {
case "blob":
return TypeBlob
case "ref":
return TypeRef
}
panic(fmt.Sprintf("unknown type %q", s))
}
func (t Type) String() string {
switch t {
case TypeBlob:
return "blob"
case TypeRef:
return "ref"
}
panic(fmt.Sprintf("unknown type %d", t))
}
// NewDirRepository creates a new dir-baked repository at the given path.
func NewDirRepository(path string) (*DirRepository, error) {
d := &DirRepository{
@ -54,7 +85,7 @@ func NewDirRepository(path string) (*DirRepository, error) {
func (r *DirRepository) create() error {
dirs := []string{
r.path,
path.Join(r.path, objectPath),
path.Join(r.path, blobPath),
path.Join(r.path, refPath),
path.Join(r.path, tempPath),
}
@ -79,10 +110,21 @@ func (r *DirRepository) Path() string {
return r.path
}
// Return temp directory in correct directory for this repository.
func (r *DirRepository) tempFile() (*os.File, error) {
return ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
}
// Rename temp file to final name according to type and ID.
func (r *DirRepository) renameFile(file *os.File, t Type, id ID) error {
filename := path.Join(r.dir(t), id.String())
return os.Rename(file.Name(), filename)
}
// Put saves content and returns the ID.
func (r *DirRepository) Put(reader io.Reader) (ID, error) {
func (r *DirRepository) Put(t Type, reader io.Reader) (ID, error) {
// save contents to tempfile, hash while writing
file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
file, err := r.tempFile()
if err != nil {
return nil, err
}
@ -100,8 +142,7 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) {
// move file to final name using hash of contents
id := ID(rd.Hash())
filename := path.Join(r.path, objectPath, id.String())
err = os.Rename(file.Name(), filename)
err = r.renameFile(file, t, id)
if err != nil {
return nil, err
}
@ -109,6 +150,23 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) {
return id, nil
}
// Construct directory for given Type.
func (r *DirRepository) dir(t Type) string {
switch t {
case TypeBlob:
return path.Join(r.path, blobPath)
case TypeRef:
return path.Join(r.path, refPath)
}
panic(fmt.Sprintf("unknown type %d", t))
}
// Construct path for given Type and ID.
func (r *DirRepository) filename(t Type, id ID) string {
return path.Join(r.dir(t), id.String())
}
// PutFile saves a file's content to the repository and returns the ID.
func (r *DirRepository) PutFile(path string) (ID, error) {
f, err := os.Open(path)
@ -117,13 +175,13 @@ func (r *DirRepository) PutFile(path string) (ID, error) {
return nil, err
}
return r.Put(f)
return r.Put(TypeBlob, f)
}
// PutRaw saves a []byte's content to the repository and returns the ID.
func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
func (r *DirRepository) PutRaw(t Type, buf []byte) (ID, error) {
// save contents to tempfile, hash while writing
file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
file, err := r.tempFile()
if err != nil {
return nil, err
}
@ -140,8 +198,7 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
// move file to final name using hash of contents
id := ID(wr.Hash())
filename := path.Join(r.path, objectPath, id.String())
err = os.Rename(file.Name(), filename)
err = r.renameFile(file, t, id)
if err != nil {
return nil, err
}
@ -150,9 +207,9 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
}
// Test returns true if the given ID exists in the repository.
func (r *DirRepository) Test(id ID) (bool, error) {
func (r *DirRepository) Test(t Type, id ID) (bool, error) {
// try to open file
file, err := os.Open(path.Join(r.path, objectPath, id.String()))
file, err := os.Open(r.filename(t, id))
defer func() {
file.Close()
}()
@ -168,9 +225,9 @@ func (r *DirRepository) Test(id ID) (bool, error) {
}
// Get returns a reader for the content stored under the given ID.
func (r *DirRepository) Get(id ID) (io.Reader, error) {
func (r *DirRepository) Get(t Type, id ID) (io.Reader, error) {
// try to open file
file, err := os.Open(path.Join(r.path, objectPath, id.String()))
file, err := os.Open(r.filename(t, id))
if err != nil {
return nil, err
}
@ -179,60 +236,66 @@ func (r *DirRepository) Get(id ID) (io.Reader, error) {
}
// Remove removes the content stored at ID.
func (r *DirRepository) Remove(id ID) error {
return os.Remove(path.Join(r.path, objectPath, id.String()))
func (r *DirRepository) Remove(t Type, id ID) error {
return os.Remove(r.filename(t, id))
}
// Unlink removes a named ID.
func (r *DirRepository) Unlink(name string) error {
return os.Remove(path.Join(r.path, refPath, Name(name).Encode()))
}
type IDs []ID
// Link assigns a name to an ID. Name must be unique in this repository and ID must exist.
func (r *DirRepository) Link(name string, id ID) error {
exist, err := r.Test(id)
if err != nil {
return err
}
// Lists all objects of a given type.
func (r *DirRepository) ListIDs(t Type) (IDs, error) {
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
pattern := path.Join(r.dir(t), "*")
if !exist {
return ErrIDDoesNotExist
}
// create file, write id
f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode()))
defer f.Close()
if err != nil {
return err
}
f.Write([]byte(hex.EncodeToString(id)))
return nil
}
// Resolve returns the ID associated with the given name.
func (r *DirRepository) Resolve(name string) (ID, error) {
f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode()))
defer f.Close()
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
// read hex string
l := r.hash().Size()
buf := make([]byte, l*2)
_, err = io.ReadFull(f, buf)
ids := make(IDs, 0, len(matches))
if err != nil {
return nil, err
for _, m := range matches {
base := filepath.Base(m)
if base == "" {
continue
}
id, err := ParseID(base)
if err != nil {
continue
}
ids = append(ids, id)
}
id := make([]byte, l)
_, err = hex.Decode(id, buf)
if err != nil {
return nil, err
}
return ID(id), nil
return ids, nil
}
func (ids IDs) Len() int {
return len(ids)
}
func (ids IDs) Less(i, j int) bool {
if len(ids[i]) < len(ids[j]) {
return true
}
for k, b := range ids[i] {
if b == ids[j][k] {
continue
}
if b < ids[j][k] {
return true
} else {
return false
}
}
return false
}
func (ids IDs) Swap(i, j int) {
ids[i], ids[j] = ids[j], ids[i]
}

66
repository_test.go
View file

@ -5,6 +5,7 @@ import (
"io"
"io/ioutil"
"os"
"sort"
"strings"
"github.com/fd0/khepri"
@ -14,12 +15,13 @@ import (
var TestStrings = []struct {
id string
t khepri.Type
data string
}{
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", khepri.TypeBlob, "foobar"},
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", khepri.TypeBlob, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", khepri.TypeRef, "foo/bar"},
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TypeBlob, "foo/../../baz"},
}
var _ = Describe("Storage", func() {
@ -30,7 +32,7 @@ var _ = Describe("Storage", func() {
id khepri.ID
)
BeforeEach(func() {
var _ = BeforeSuite(func() {
tempdir, err = ioutil.TempDir("", "khepri-test-")
if err != nil {
panic(err)
@ -41,7 +43,7 @@ var _ = Describe("Storage", func() {
}
})
AfterEach(func() {
AfterSuite(func() {
err = os.RemoveAll(tempdir)
if err != nil {
panic(err)
@ -58,7 +60,7 @@ var _ = Describe("Storage", func() {
Expect(err).NotTo(HaveOccurred())
// try to get string out, should fail
ret, err := repo.Test(id)
ret, err := repo.Test(test.t, id)
Expect(ret).Should(Equal(false))
}
})
@ -66,44 +68,62 @@ var _ = Describe("Storage", func() {
It("Should Add File", func() {
for _, test := range TestStrings {
// store string in repository
id, err = repo.Put(strings.NewReader(test.data))
id, err = repo.Put(test.t, strings.NewReader(test.data))
Expect(err).NotTo(HaveOccurred())
Expect(id.String()).Should(Equal(test.id))
// try to get it out again
var buf bytes.Buffer
rd, err := repo.Get(id)
rd, err := repo.Get(test.t, id)
Expect(err).NotTo(HaveOccurred())
Expect(rd).ShouldNot(BeNil())
// compare content
Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data))))
Expect(buf.Bytes()).Should(Equal([]byte(test.data)))
// store id under name
err = repo.Link(test.data, id)
Expect(err).NotTo(HaveOccurred())
// resolve again
Expect(repo.Resolve(test.data)).Should(Equal(id))
// remove link
Expect(repo.Unlink(test.data)).NotTo(HaveOccurred())
// remove string
Expect(repo.Remove(id))
}
})
It("Should Add Buffer", func() {
for _, test := range TestStrings {
// store buf in repository
id, err := repo.PutRaw([]byte(test.data))
id, err := repo.PutRaw(test.t, []byte(test.data))
Expect(err).NotTo(HaveOccurred())
Expect(id.String()).To(Equal(test.id))
}
})
It("Should List IDs", func() {
for _, t := range []khepri.Type{khepri.TypeBlob, khepri.TypeRef} {
IDs := khepri.IDs{}
for _, test := range TestStrings {
if test.t == t {
id, err := khepri.ParseID(test.id)
Expect(err).NotTo(HaveOccurred())
IDs = append(IDs, id)
}
}
ids, err := repo.ListIDs(t)
sort.Sort(ids)
sort.Sort(IDs)
Expect(err).NotTo(HaveOccurred())
Expect(ids).Should(Equal(IDs))
}
})
It("Should Remove Content", func() {
for _, test := range TestStrings {
id, err := khepri.ParseID(test.id)
Expect(err).ShouldNot(HaveOccurred())
Expect(repo.Test(test.t, id)).To(Equal(true))
Expect(repo.Remove(test.t, id))
Expect(repo.Test(test.t, id)).To(Equal(false))
}
})
})
})
})