repository: test uncompressedLength field and index example

internal/repository/index_test.go

@@ -23,11 +23,17 @@ func TestIndexSerialize(t *testing.T) {
 		pos := uint(0)
 		for j := 0; j < 20; j++ {
 			length := uint(i*100 + j)
+			uncompressedLength := uint(0)
+			if i >= 25 {
+				// test a mix of compressed and uncompressed packs
+				uncompressedLength = 2 * length
+			}
 			pb := restic.PackedBlob{
 				Blob: restic.Blob{
 					BlobHandle:         restic.NewRandomBlobHandle(),
 					Offset:             pos,
 					Length:             length,
+					UncompressedLength: uncompressedLength,
 				},
 				PackID: packID,
 			}
@@ -164,7 +170,7 @@ func TestIndexSize(t *testing.T) {
 }
 
 // example index serialization from doc/Design.rst
-var docExample = []byte(`
+var docExampleV1 = []byte(`
 {
   "supersedes": [
 	"ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
@@ -177,12 +183,12 @@ var docExample = []byte(`
 		  "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
 		  "type": "data",
 		  "offset": 0,
-		  "length": 25
+		  "length": 38
 		},{
 		  "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
 		  "type": "tree",
 		  "offset": 38,
-		  "length": 100
+		  "length": 112
 		},
 		{
 		  "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
@@ -196,6 +202,41 @@ var docExample = []byte(`
 }
 `)
 
+var docExampleV2 = []byte(`
+{
+	"supersedes": [
+	  "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
+	],
+	"packs": [
+	  {
+		"id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
+		"blobs": [
+		  {
+			"id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
+			"type": "data",
+			"offset": 0,
+			"length": 38
+		  },
+		  {
+			"id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
+			"type": "tree",
+			"offset": 38,
+			"length": 112,
+			"uncompressed_length": 511
+		  },
+		  {
+			"id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
+			"type": "data",
+			"offset": 150,
+			"length": 123,
+			"uncompressed_length": 234
+		  }
+		]
+	  }
+	]
+  }
+`)
+
 var docOldExample = []byte(`
 [ {
   "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
@@ -204,12 +245,12 @@ var docOldExample = []byte(`
 	  "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
 	  "type": "data",
 	  "offset": 0,
-	  "length": 25
+	  "length": 38
 	},{
 	  "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
 	  "type": "tree",
 	  "offset": 38,
-	  "length": 100
+	  "length": 112
 	},
 	{
 	  "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
@@ -225,19 +266,20 @@ var exampleTests = []struct {
 	id, packID         restic.ID
 	tpe                restic.BlobType
 	offset, length     uint
+	uncompressedLength uint
 }{
 	{
 		restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"),
 		restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
-		restic.DataBlob, 0, 25,
+		restic.DataBlob, 0, 38, 0,
 	}, {
 		restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"),
 		restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
-		restic.TreeBlob, 38, 100,
+		restic.TreeBlob, 38, 112, 511,
 	}, {
 		restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"),
 		restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
-		restic.DataBlob, 150, 123,
+		restic.DataBlob, 150, 123, 234,
 	},
 }
 
@@ -254,9 +296,16 @@ var exampleLookupTest = struct {
 }
 
 func TestIndexUnserialize(t *testing.T) {
+	for _, task := range []struct {
+		idxBytes []byte
+		version  int
+	}{
+		{docExampleV1, 1},
+		{docExampleV2, 2},
+	} {
 		oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
 
-	idx, oldFormat, err := repository.DecodeIndex(docExample, restic.NewRandomID())
+		idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID())
 		rtest.OK(t, err)
 		rtest.Assert(t, !oldFormat, "new index format recognized as old format")
 
@@ -273,6 +322,13 @@ func TestIndexUnserialize(t *testing.T) {
 			rtest.Equals(t, test.tpe, blob.Type)
 			rtest.Equals(t, test.offset, blob.Offset)
 			rtest.Equals(t, test.length, blob.Length)
+			if task.version == 1 {
+				rtest.Equals(t, uint(0), blob.UncompressedLength)
+			} else if task.version == 2 {
+				rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength)
+			} else {
+				t.Fatal("Invalid index version")
+			}
 		}
 
 		rtest.Equals(t, oldIdx, idx.Supersedes())
@@ -292,6 +348,7 @@ func TestIndexUnserialize(t *testing.T) {
 			}
 		}
 	}
+}
 
 var (
 	benchmarkIndexJSON     []byte
@@ -404,6 +461,7 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lo
 					ID:   id,
 				},
 				Length:             uint(size),
+				UncompressedLength: uint(2 * size),
 				Offset:             uint(offset),
 			})
 
@@ -475,11 +533,17 @@ func TestIndexHas(t *testing.T) {
 		pos := uint(0)
 		for j := 0; j < 20; j++ {
 			length := uint(i*100 + j)
+			uncompressedLength := uint(0)
+			if i >= 25 {
+				// test a mix of compressed and uncompressed packs
+				uncompressedLength = 2 * length
+			}
 			pb := restic.PackedBlob{
 				Blob: restic.Blob{
 					BlobHandle:         restic.NewRandomBlobHandle(),
 					Offset:             pos,
 					Length:             length,
+					UncompressedLength: uncompressedLength,
 				},
 				PackID: packID,
 			}

internal/repository/master_index_test.go

@@ -33,6 +33,7 @@ func TestMasterIndex(t *testing.T) {
 			BlobHandle:         bhInIdx2,
 			Length:             uint(restic.CiphertextLength(100)),
 			Offset:             10,
+			UncompressedLength: 200,
 		},
 	}
 
@@ -51,6 +52,7 @@ func TestMasterIndex(t *testing.T) {
 			BlobHandle:         bhInIdx12,
 			Length:             uint(restic.CiphertextLength(123)),
 			Offset:             50,
+			UncompressedLength: 80,
 		},
 	}
 
@@ -86,7 +88,7 @@ func TestMasterIndex(t *testing.T) {
 
 	size, found = mIdx.LookupSize(bhInIdx2)
 	rtest.Equals(t, true, found)
-	rtest.Equals(t, uint(100), size)
+	rtest.Equals(t, uint(200), size)
 
 	// test idInIdx12
 	found = mIdx.Has(bhInIdx12)
@@ -147,6 +149,7 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
 			BlobHandle:         bhInIdx2,
 			Length:             100,
 			Offset:             10,
+			UncompressedLength: 200,
 		},
 	}