From 51aff3ca571abb1abee27d0a2f3eee9908b68b04 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 1 Nov 2015 22:45:52 +0100
Subject: [PATCH] Add FindBlobsForPacks()
---
checker/repacker.go | 26 +++++++++++----
checker/repacker_test.go | 69 ++++++++++++++++++++++++++++++----------
2 files changed, 73 insertions(+), 22 deletions(-)
diff --git a/checker/repacker.go b/checker/repacker.go
index 39fd47355..e2669fa03 100644
--- a/checker/repacker.go
+++ b/checker/repacker.go
@@ -11,14 +11,14 @@ import (
// Repacker extracts still used blobs from packs with unused blobs and creates
// new packs.
type Repacker struct {
- unusedBlobs []backend.ID
+ unusedBlobs backend.IDSet
src, dst *repository.Repository
}
// NewRepacker returns a new repacker that (when Repack() in run) cleans up the
// repository and creates new packs and indexs so that all blobs in unusedBlobs
// aren't used any more.
-func NewRepacker(src, dst *repository.Repository, unusedBlobs []backend.ID) *Repacker {
+func NewRepacker(src, dst *repository.Repository, unusedBlobs backend.IDSet) *Repacker {
return &Repacker{
src: src,
dst: dst,
@@ -30,7 +30,8 @@ func NewRepacker(src, dst *repository.Repository, unusedBlobs []backend.ID) *Rep
// blobs, extracts them and creates new packs with just the still-in-use blobs.
func (r *Repacker) Repack() error {
debug.Log("Repacker.Repack", "searching packs for %v", r.unusedBlobs)
- packs, err := FindPacksforBlobs(r.src, r.unusedBlobs)
+
+ packs, err := FindPacksForBlobs(r.src, r.unusedBlobs)
if err != nil {
return err
}
@@ -40,11 +41,11 @@ func (r *Repacker) Repack() error {
return nil
}
-// FindPacksforBlobs returns the set of packs that contain the blobs.
-func FindPacksforBlobs(repo *repository.Repository, blobs []backend.ID) (backend.IDSet, error) {
+// FindPacksForBlobs returns the set of packs that contain the blobs.
+func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) {
packs := backend.NewIDSet()
idx := repo.Index()
- for _, id := range blobs {
+ for id := range blobs {
blob, err := idx.Lookup(id)
if err != nil {
return nil, err
@@ -56,6 +57,19 @@ func FindPacksforBlobs(repo *repository.Repository, blobs []backend.ID) (backend
return packs, nil
}
+// FindBlobsForPacks returns the set of blobs contained in a pack of packs.
+func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) {
+ blobs := backend.NewIDSet()
+
+ for packID := range packs {
+ for _, packedBlob := range repo.Index().ListPack(packID) {
+ blobs.Insert(packedBlob.ID)
+ }
+ }
+
+ return blobs, nil
+}
+
// repackBlob loads a single blob from src and saves it in dst.
func repackBlob(src, dst *repository.Repository, id backend.ID) error {
blob, err := src.Index().Lookup(id)
diff --git a/checker/repacker_test.go b/checker/repacker_test.go
index c0e70ae12..a2e9979ce 100644
--- a/checker/repacker_test.go
+++ b/checker/repacker_test.go
@@ -10,15 +10,15 @@ import (
)
var findPackTests = []struct {
- blobIDs backend.IDs
+ blobIDs backend.IDSet
packIDs backend.IDSet
}{
{
- backend.IDs{
- ParseID("534f211b4fc2cf5b362a24e8eba22db5372a75b7e974603ff9263f5a471760f4"),
- ParseID("51aa04744b518c6a85b4e7643cfa99d58789c2a6ca2a3fda831fa3032f28535c"),
- ParseID("454515bca5f4f60349a527bd814cc2681bc3625716460cc6310771c966d8a3bf"),
- ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"),
+ backend.IDSet{
+ ParseID("534f211b4fc2cf5b362a24e8eba22db5372a75b7e974603ff9263f5a471760f4"): struct{}{},
+ ParseID("51aa04744b518c6a85b4e7643cfa99d58789c2a6ca2a3fda831fa3032f28535c"): struct{}{},
+ ParseID("454515bca5f4f60349a527bd814cc2681bc3625716460cc6310771c966d8a3bf"): struct{}{},
+ ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"): struct{}{},
},
backend.IDSet{
ParseID("19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068"): struct{}{},
@@ -27,6 +27,41 @@ var findPackTests = []struct {
},
}
+var findBlobTests = []struct {
+ packIDs backend.IDSet
+ blobIDs backend.IDSet
+}{
+ {
+ backend.IDSet{
+ ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{},
+ },
+ backend.IDSet{
+ ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{},
+ ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{},
+ ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{},
+ ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{},
+ ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{},
+ },
+ },
+ {
+ backend.IDSet{
+ ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{},
+ ParseID("ff7e12cd66d896b08490e787d1915c641e678d7e6b4a00e60db5d13054f4def4"): struct{}{},
+ },
+ backend.IDSet{
+ ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{},
+ ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{},
+ ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{},
+ ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{},
+ ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{},
+ ParseID("aa79d596dbd4c863e5400deaca869830888fe1ce9f51b4a983f532c77f16a596"): struct{}{},
+ ParseID("b2396c92781307111accf2ebb1cd62b58134b744d90cb6f153ca456a98dc3e76"): struct{}{},
+ ParseID("5249af22d3b2acd6da8048ac37b2a87fa346fabde55ed23bb866f7618843c9fe"): struct{}{},
+ ParseID("f41c2089a9d58a4b0bf39369fa37588e6578c928aea8e90a4490a6315b9905c1"): struct{}{},
+ },
+ },
+}
+
func TestRepackerFindPacks(t *testing.T) {
WithTestEnvironment(t, checkerTestData, func(repodir string) {
repo := OpenLocalRepo(t, repodir)
@@ -34,10 +69,19 @@ func TestRepackerFindPacks(t *testing.T) {
OK(t, repo.LoadIndex())
for _, test := range findPackTests {
- packIDs, err := checker.FindPacksforBlobs(repo, test.blobIDs)
+ packIDs, err := checker.FindPacksForBlobs(repo, test.blobIDs)
OK(t, err)
Equals(t, test.packIDs, packIDs)
}
+
+ for _, test := range findBlobTests {
+ blobs, err := checker.FindBlobsForPacks(repo, test.packIDs)
+ OK(t, err)
+
+ Assert(t, test.blobIDs.Equals(blobs),
+ "list of blobs for packs %v does not match, expected:\n %v\ngot:\n %v",
+ test.packIDs, test.blobIDs, blobs)
+ }
})
}
@@ -67,15 +111,8 @@ func TestRepackBlobs(t *testing.T) {
t.Fatalf("expected unused blobs:\n %v\ngot:\n %v", unusedBlobs, list)
}
- // repacker := checker.NewRepacker(repo, repo, repackBlobIDs)
- // OK(t, repacker.Repack())
-
- // err := checker.RepackBlobs(repo, repo, repackBlobIDs)
- // OK(t, err)
-
- // newPackIDs, err := checker.FindPacksforBlobs(repo, repackBlobIDs)
- // OK(t, err)
- // fmt.Printf("new pack IDs: %v\n", newPackIDs)
+ repacker := checker.NewRepacker(repo, repo, unusedBlobs)
+ OK(t, repacker.Repack())
chkr = checker.New(repo)
_, errs = chkr.LoadIndex()