From 33ce4e33aa49997064cb937dffc828b193e36e70 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 23 Nov 2014 22:26:01 +0100
Subject: [PATCH] Store maps in new subdir "maps"
---
archiver.go | 11 ++++++++---
backend/interface.go | 1 +
backend/local.go | 4 ++++
backend/local_test.go | 2 +-
backend/sftp.go | 2 ++
bloblist.go | 12 ++++++++++++
cmd/khepri/cmd_cat.go | 17 ++++++++++++++++-
contenthandler.go | 17 +++++++++++------
snapshot.go | 3 ++-
9 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/archiver.go b/archiver.go
index b1c593ab8..71dc988d2 100644
--- a/archiver.go
+++ b/archiver.go
@@ -85,7 +85,7 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
}
// load all blobs from all snapshots
- err = arch.ch.LoadAllSnapshots()
+ err = arch.ch.LoadAllMaps()
if err != nil {
return nil, err
}
@@ -371,11 +371,16 @@ func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, erro
if err != nil {
return nil, nil, err
}
-
sn.Content = blob.ID
+ // save bloblist
+ blob, err = arch.SaveJSON(backend.Map, arch.bl)
+ if err != nil {
+ return nil, nil, err
+ }
+ sn.Map = blob.Storage
+
// save snapshot
- sn.BlobList = arch.bl
blob, err = arch.SaveJSON(backend.Snapshot, sn)
if err != nil {
return nil, nil, err
diff --git a/backend/interface.go b/backend/interface.go
index b45f962f0..bfaabacb6 100644
--- a/backend/interface.go
+++ b/backend/interface.go
@@ -10,6 +10,7 @@ const (
Lock = "lock"
Snapshot = "snapshot"
Tree = "tree"
+ Map = "map"
)
const (
diff --git a/backend/local.go b/backend/local.go
index 9ba203fbc..b7e1508d1 100644
--- a/backend/local.go
+++ b/backend/local.go
@@ -17,6 +17,7 @@ const (
dataPath = "data"
snapshotPath = "snapshots"
treePath = "trees"
+ mapPath = "maps"
lockPath = "locks"
keyPath = "keys"
tempPath = "tmp"
@@ -86,6 +87,7 @@ func CreateLocal(dir string) (*Local, error) {
filepath.Join(dir, dataPath),
filepath.Join(dir, snapshotPath),
filepath.Join(dir, treePath),
+ filepath.Join(dir, mapPath),
filepath.Join(dir, lockPath),
filepath.Join(dir, keyPath),
filepath.Join(dir, tempPath),
@@ -158,6 +160,8 @@ func (b *Local) dir(t Type) string {
n = snapshotPath
case Tree:
n = treePath
+ case Map:
+ n = mapPath
case Lock:
n = lockPath
case Key:
diff --git a/backend/local_test.go b/backend/local_test.go
index 056ba916f..0c309c063 100644
--- a/backend/local_test.go
+++ b/backend/local_test.go
@@ -45,7 +45,7 @@ func teardownBackend(t *testing.T, b *backend.Local) {
}
func testBackend(b backend.Server, t *testing.T) {
- for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
+ for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
// detect non-existing files
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
diff --git a/backend/sftp.go b/backend/sftp.go
index ff5774295..0299cd133 100644
--- a/backend/sftp.go
+++ b/backend/sftp.go
@@ -238,6 +238,8 @@ func (r *SFTP) dir(t Type) string {
n = snapshotPath
case Tree:
n = treePath
+ case Map:
+ n = mapPath
case Lock:
n = lockPath
case Key:
diff --git a/bloblist.go b/bloblist.go
index 48820e95f..07da14cfa 100644
--- a/bloblist.go
+++ b/bloblist.go
@@ -6,6 +6,8 @@ import (
"errors"
"sort"
"sync"
+
+ "github.com/fd0/khepri/backend"
)
type BlobList struct {
@@ -21,6 +23,16 @@ func NewBlobList() *BlobList {
}
}
+func LoadBlobList(ch *ContentHandler, id backend.ID) (*BlobList, error) {
+ bl := &BlobList{}
+ err := ch.LoadJSONRaw(backend.Map, id, bl)
+ if err != nil {
+ return nil, err
+ }
+
+ return bl, nil
+}
+
func (bl *BlobList) find(blob Blob) (int, Blob, error) {
pos := sort.Search(len(bl.list), func(i int) bool {
return blob.ID.Compare(bl.list[i].ID) >= 0
diff --git a/cmd/khepri/cmd_cat.go b/cmd/khepri/cmd_cat.go
index 0e3b67056..19d925900 100644
--- a/cmd/khepri/cmd_cat.go
+++ b/cmd/khepri/cmd_cat.go
@@ -27,7 +27,7 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
return err
}
- err = ch.LoadAllSnapshots()
+ err = ch.LoadAllMaps()
if err != nil {
return err
}
@@ -87,6 +87,21 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
fmt.Println(string(buf))
+ return nil
+ case "map":
+ var bl khepri.BlobList
+ err := ch.LoadJSONRaw(backend.Map, id, &bl)
+ if err != nil {
+ return err
+ }
+
+ buf, err := json.MarshalIndent(&bl, "", " ")
+ if err != nil {
+ return err
+ }
+
+ fmt.Println(string(buf))
+
return nil
case "snapshot":
var sn khepri.Snapshot
diff --git a/contenthandler.go b/contenthandler.go
index 8ccf7654f..c4f802862 100644
--- a/contenthandler.go
+++ b/contenthandler.go
@@ -33,22 +33,27 @@ func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) {
return nil, err
}
- ch.bl.Merge(sn.BlobList)
+ sn.bl, err = LoadBlobList(ch, sn.Map)
+ if err != nil {
+ return nil, err
+ }
+
+ ch.bl.Merge(sn.bl)
return sn, nil
}
-// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted
+// LoadAllMaps adds all blobs from all snapshots that can be decrypted
// into the content handler.
-func (ch *ContentHandler) LoadAllSnapshots() error {
+func (ch *ContentHandler) LoadAllMaps() error {
// add all maps from all snapshots that can be decrypted to the storage map
- err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) {
- sn, err := LoadSnapshot(ch, id)
+ err := backend.EachID(ch.be, backend.Map, func(id backend.ID) {
+ bl, err := LoadBlobList(ch, id)
if err != nil {
return
}
- ch.bl.Merge(sn.BlobList)
+ ch.bl.Merge(bl)
})
if err != nil {
return err
diff --git a/snapshot.go b/snapshot.go
index c0cc6c2bb..a92dd19ff 100644
--- a/snapshot.go
+++ b/snapshot.go
@@ -13,7 +13,7 @@ import (
type Snapshot struct {
Time time.Time `json:"time"`
Content backend.ID `json:"content"`
- BlobList *BlobList `json:"blobs"`
+ Map backend.ID `json:"map"`
Dir string `json:"dir"`
Hostname string `json:"hostname,omitempty"`
Username string `json:"username,omitempty"`
@@ -21,6 +21,7 @@ type Snapshot struct {
GID string `json:"gid,omitempty"`
id backend.ID // plaintext ID, used during restore
+ bl *BlobList
}
func NewSnapshot(dir string) *Snapshot {