restic/internal/restic/tree_test.go

package restic_test

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/restic/restic/internal/archiver"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"
	rtest "github.com/restic/restic/internal/test"
	"golang.org/x/sync/errgroup"
)

var testFiles = []struct {
	name    string
	content []byte
}{
	{"foo", []byte("bar")},
	{"bar/foo2", []byte("bar2")},
	{"bar/bla/blubb", []byte("This is just a test!\n")},
}

func createTempDir(t *testing.T) string {
	tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
	rtest.OK(t, err)

	for _, test := range testFiles {
		file := filepath.Join(tempdir, test.name)
		dir := filepath.Dir(file)
		if dir != "." {
			rtest.OK(t, os.MkdirAll(dir, 0755))
		}

		f, err := os.Create(file)
		defer func() {
			rtest.OK(t, f.Close())
		}()

		rtest.OK(t, err)

		_, err = f.Write(test.content)
		rtest.OK(t, err)
	}

	return tempdir
}

func TestTree(t *testing.T) {
	dir := createTempDir(t)
	defer func() {
		if rtest.TestCleanupTempDirs {
			rtest.RemoveAll(t, dir)
		}
	}()
}

var testNodes = []restic.Node{
	{Name: "normal"},
	{Name: "with backslashes \\zzz"},
	{Name: "test utf-8 föbärß"},
	{Name: "test invalid \x00\x01\x02\x03\x04"},
	{Name: "test latin1 \x75\x6d\x6c\xe4\xfc\x74\xf6\x6e\xdf\x6e\x6c\x6c"},
}

func TestNodeMarshal(t *testing.T) {
	for i, n := range testNodes {
		data, err := json.Marshal(&n)
		rtest.OK(t, err)

		var node restic.Node
		err = json.Unmarshal(data, &node)
		rtest.OK(t, err)

		if n.Name != node.Name {
			t.Fatalf("Node %d: Names are not equal, want: %q got: %q", i, n.Name, node.Name)
		}
	}
}

func nodeForFile(t *testing.T, name string) *restic.Node {
	f, err := (&fs.Local{}).OpenFile(name, fs.O_NOFOLLOW, true)
	rtest.OK(t, err)
	node, err := f.ToNode(false)
	rtest.OK(t, err)
	rtest.OK(t, f.Close())
	return node
}

func TestNodeComparison(t *testing.T) {
	node := nodeForFile(t, "tree_test.go")

	n2 := *node
	rtest.Assert(t, node.Equals(n2), "nodes aren't equal")

	n2.Size--
	rtest.Assert(t, !node.Equals(n2), "nodes are equal")
}

func TestEmptyLoadTree(t *testing.T) {
	repo := repository.TestRepository(t)

	var wg errgroup.Group
	repo.StartPackUploader(context.TODO(), &wg)
	// save tree
	tree := restic.NewTree(0)
	id, err := restic.SaveTree(context.TODO(), repo, tree)
	rtest.OK(t, err)

	// save packs
	rtest.OK(t, repo.Flush(context.Background()))

	// load tree again
	tree2, err := restic.LoadTree(context.TODO(), repo, id)
	rtest.OK(t, err)

	rtest.Assert(t, tree.Equals(tree2),
		"trees are not equal: want %v, got %v",
		tree, tree2)
}

func TestTreeEqualSerialization(t *testing.T) {
	files := []string{"node.go", "tree.go", "tree_test.go"}
	for i := 1; i <= len(files); i++ {
		tree := restic.NewTree(i)
		builder := restic.NewTreeJSONBuilder()

		for _, fn := range files[:i] {
			node := nodeForFile(t, fn)

			rtest.OK(t, tree.Insert(node))
			rtest.OK(t, builder.AddNode(node))

			rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node")
			rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node")
			rtest.Assert(t, errors.Is(builder.AddNode(node), restic.ErrTreeNotOrdered), "wrong error returned")
		}

		treeBytes, err := json.Marshal(tree)
		treeBytes = append(treeBytes, '\n')
		rtest.OK(t, err)

		stiBytes, err := builder.Finalize()
		rtest.OK(t, err)

		// compare serialization of an individual node and the SaveTreeIterator
		rtest.Equals(t, treeBytes, stiBytes)
	}
}

func BenchmarkBuildTree(b *testing.B) {
	const size = 100 // Directories of this size are not uncommon.

	nodes := make([]restic.Node, size)
	for i := range nodes {
		// Archiver.SaveTree inputs in sorted order, so do that here too.
		nodes[i].Name = strconv.Itoa(i)
	}

	b.ResetTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		t := restic.NewTree(size)

		for i := range nodes {
			_ = t.Insert(&nodes[i])
		}
	}
}

func TestLoadTree(t *testing.T) {
	repository.TestAllVersions(t, testLoadTree)
}

func testLoadTree(t *testing.T, version uint) {
	if rtest.BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping")
	}

	// archive a few files
	repo, _ := repository.TestRepositoryWithVersion(t, version)
	sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
	rtest.OK(t, repo.Flush(context.Background()))

	_, err := restic.LoadTree(context.TODO(), repo, *sn.Tree)
	rtest.OK(t, err)
}

func BenchmarkLoadTree(t *testing.B) {
	repository.BenchmarkAllVersions(t, benchmarkLoadTree)
}

func benchmarkLoadTree(t *testing.B, version uint) {
	if rtest.BenchArchiveDirectory == "" {
		t.Skip("benchdir not set, skipping")
	}

	// archive a few files
	repo, _ := repository.TestRepositoryWithVersion(t, version)
	sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
	rtest.OK(t, repo.Flush(context.Background()))

	t.ResetTimer()

	for i := 0; i < t.N; i++ {
		_, err := restic.LoadTree(context.TODO(), repo, *sn.Tree)
		rtest.OK(t, err)
	}
}

func TestFindTreeDirectory(t *testing.T) {
	repo := repository.TestRepository(t)
	sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3)

	for _, exp := range []struct {
		subfolder string
		id        restic.ID
		err       error
	}{
		{"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
		{"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
		{".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
		{"..", restic.ID{}, errors.New("path ..: not found")},
		{"file-1", restic.ID{}, errors.New("path file-1: not a directory")},
		{"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
		{"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
		{"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
		{"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil},
	} {
		t.Run("", func(t *testing.T) {
			id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder)
			if exp.err == nil {
				rtest.OK(t, err)
				rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id)
			} else {
				rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err)
			}
		})
	}

	_, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "")
	rtest.Assert(t, err != nil, "missing error on null tree id")
}