forked from TrueCloudLab/restic
Add storage
This commit is contained in:
Alexander Neumann
parent
e3e77d4e9a
commit
d156b2d245
3 changed files with 342 additions and 0 deletions
231
storage/storage.go
Normal file
231
storage/storage.go
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/fd0/khepri/hashing"
|
||||
)
|
||||
|
||||
const (
|
||||
dirMode = 0700
|
||||
objectPath = "objects"
|
||||
refPath = "refs"
|
||||
tempPath = "tmp"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
Put(reader io.Reader) (ID, error)
|
||||
PutFile(path string) (ID, error)
|
||||
Get(ID) (io.Reader, error)
|
||||
Test(ID) (bool, error)
|
||||
Link(name string, id ID) error
|
||||
Unlink(name string) error
|
||||
Resolve(name string) (ID, error)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrIDDoesNotExist = errors.New("ID does not exist")
|
||||
)
|
||||
|
||||
// References content within a repository.
|
||||
type ID []byte
|
||||
|
||||
func (id ID) String() string {
|
||||
return hex.EncodeToString(id)
|
||||
}
|
||||
|
||||
// Equal compares an ID to another other.
|
||||
func (id ID) Equal(other ID) bool {
|
||||
return bytes.Equal(id, other)
|
||||
}
|
||||
|
||||
// EqualString compares this ID to another one, given as a string.
|
||||
func (id ID) EqualString(other string) (bool, error) {
|
||||
s, err := hex.DecodeString(other)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return id.Equal(ID(s)), nil
|
||||
}
|
||||
|
||||
// Name stands for the alias given to an ID.
|
||||
type Name string
|
||||
|
||||
func (n Name) Encode() string {
|
||||
return url.QueryEscape(string(n))
|
||||
}
|
||||
|
||||
type Dir struct {
|
||||
path string
|
||||
hash func() hash.Hash
|
||||
}
|
||||
|
||||
// NewDir creates a new dir-baked repository at the given path.
|
||||
func NewDir(path string) (*Dir, error) {
|
||||
d := &Dir{
|
||||
path: path,
|
||||
hash: sha256.New,
|
||||
}
|
||||
|
||||
err := d.create()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (r *Dir) create() error {
|
||||
dirs := []string{
|
||||
r.path,
|
||||
path.Join(r.path, objectPath),
|
||||
path.Join(r.path, refPath),
|
||||
path.Join(r.path, tempPath),
|
||||
}
|
||||
|
||||
for _, dir := range dirs {
|
||||
err := os.MkdirAll(dir, dirMode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetHash changes the hash function used for deriving IDs. Default is SHA256.
|
||||
func (r *Dir) SetHash(h func() hash.Hash) {
|
||||
r.hash = h
|
||||
}
|
||||
|
||||
// Path returns the directory used for this repository.
|
||||
func (r *Dir) Path() string {
|
||||
return r.path
|
||||
}
|
||||
|
||||
// Put saves content and returns the ID.
|
||||
func (r *Dir) Put(reader io.Reader) (ID, error) {
|
||||
// save contents to tempfile, hash while writing
|
||||
file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rd := hashing.NewReader(reader, r.hash)
|
||||
_, err = io.Copy(file, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// move file to final name using hash of contents
|
||||
id := ID(rd.Hash())
|
||||
filename := path.Join(r.path, objectPath, id.String())
|
||||
err = os.Rename(file.Name(), filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// PutFile saves a file's content to the repository and returns the ID.
|
||||
func (r *Dir) PutFile(path string) (ID, error) {
|
||||
f, err := os.Open(path)
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return r.Put(f)
|
||||
}
|
||||
|
||||
// Test returns true if the given ID exists in the repository.
|
||||
func (r *Dir) Test(id ID) (bool, error) {
|
||||
// try to open file
|
||||
file, err := os.Open(path.Join(r.path, objectPath, id.String()))
|
||||
defer func() {
|
||||
file.Close()
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Get returns a reader for the content stored under the given ID.
|
||||
func (r *Dir) Get(id ID) (io.Reader, error) {
|
||||
// try to open file
|
||||
file, err := os.Open(path.Join(r.path, objectPath, id.String()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Unlink removes a named ID.
|
||||
func (r *Dir) Unlink(name string) error {
|
||||
return os.Remove(path.Join(r.path, refPath, Name(name).Encode()))
|
||||
}
|
||||
|
||||
// Link assigns a name to an ID. Name must be unique in this repository and ID must exist.
|
||||
func (r *Dir) Link(name string, id ID) error {
|
||||
exist, err := r.Test(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exist {
|
||||
return ErrIDDoesNotExist
|
||||
}
|
||||
|
||||
// create file, write id
|
||||
f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode()))
|
||||
defer f.Close()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.Write(id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Resolve returns the ID associated with the given name.
|
||||
func (r *Dir) Resolve(name string) (ID, error) {
|
||||
f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode()))
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := make([]byte, r.hash().Size())
|
||||
_, err = io.ReadFull(f, id)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ID(id), nil
|
||||
}
|
||||
13
storage/storage_suite_test.go
Normal file
13
storage/storage_suite_test.go
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
package storage_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStorage(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Storage Suite")
|
||||
}
|
||||
98
storage/storage_test.go
Normal file
98
storage/storage_test.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package storage_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/fd0/khepri/storage"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var TestStrings = []struct {
|
||||
id string
|
||||
data string
|
||||
}{
|
||||
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
|
||||
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
|
||||
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
|
||||
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
|
||||
}
|
||||
|
||||
var _ = Describe("Storage", func() {
|
||||
var (
|
||||
tempdir string
|
||||
repo storage.Repository
|
||||
err error
|
||||
id storage.ID
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tempdir, err = ioutil.TempDir("", "khepri-test-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
repo, err = storage.NewDir(tempdir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
err = os.RemoveAll(tempdir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// fmt.Fprintf(os.Stderr, "leaving tempdir %s", tempdir)
|
||||
tempdir = ""
|
||||
})
|
||||
|
||||
Describe("Repository", func() {
|
||||
Context("File Operations", func() {
|
||||
It("Should detect non-existing file", func() {
|
||||
for _, test := range TestStrings {
|
||||
id, err := hex.DecodeString(test.id)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// try to get string out, should fail
|
||||
ret, err := repo.Test(id)
|
||||
Expect(ret).Should(Equal(false))
|
||||
}
|
||||
})
|
||||
|
||||
It("Should Add File", func() {
|
||||
for _, test := range TestStrings {
|
||||
// store string in repository
|
||||
id, err = repo.Put(strings.NewReader(test.data))
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(id.String()).Should(Equal(test.id))
|
||||
|
||||
// try to get it out again
|
||||
var buf bytes.Buffer
|
||||
rd, err := repo.Get(id)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(rd).ShouldNot(BeNil())
|
||||
|
||||
// compare content
|
||||
Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data))))
|
||||
Expect(buf.Bytes()).Should(Equal([]byte(test.data)))
|
||||
|
||||
// store id under name
|
||||
err = repo.Link(test.data, id)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// resolve again
|
||||
Expect(repo.Resolve(test.data)).Should(Equal(id))
|
||||
|
||||
// remove link
|
||||
Expect(repo.Unlink(test.data)).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Reference in a new issue