From 9a2b85d71c21bed57d395bd1e9c2fa858cb6497a Mon Sep 17 00:00:00 2001 From: Dan McArdle Date: Thu, 11 Apr 2024 11:22:46 -0400 Subject: [PATCH] cmd/gitannex: Refactor e2e tests, add layout compat tests TestEndToEndRepoLayoutCompat exercises git-annex-remote-rclone-builtin and git-annex-remote-rclone on the same rclone remote to ensure they are compatible. It repeats the same test for all known layout modes. Issue #7625 --- cmd/gitannex/e2e_test.go | 313 ++++++++++++++++++++++++++++++--------- 1 file changed, 247 insertions(+), 66 deletions(-) diff --git a/cmd/gitannex/e2e_test.go b/cmd/gitannex/e2e_test.go index 87704df87..c989168a2 100644 --- a/cmd/gitannex/e2e_test.go +++ b/cmd/gitannex/e2e_test.go @@ -1,6 +1,7 @@ package gitannex import ( + "bytes" "encoding/json" "errors" "fmt" @@ -50,25 +51,137 @@ func checkRcloneBinaryVersion() error { return nil } -// This end-to-end test runs `git annex testremote` in a temporary git repo. -// This test will be skipped unless the `rclone` binary on PATH reports the -// expected version. +// countFilesRecursively returns the number of files nested underneath `dir`. It +// counts files only and excludes directories. +func countFilesRecursively(t *testing.T, dir string) int { + remoteFiles, err := os.ReadDir(dir) + require.NoError(t, err) + + var count int + for _, f := range remoteFiles { + if f.IsDir() { + subdir := filepath.Join(dir, f.Name()) + count += countFilesRecursively(t, subdir) + } else { + count++ + } + } + return count +} + +func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool { + remoteFiles, err := os.ReadDir(dir) + require.NoError(t, err) + + for _, f := range remoteFiles { + fPath := filepath.Join(dir, f.Name()) + if f.IsDir() { + if findFileWithContents(t, fPath, wantContents) { + return true + } + } else { + contents, err := os.ReadFile(fPath) + require.NoError(t, err) + if bytes.Equal(contents, wantContents) { + return true + } + } + } + return false +} + +type e2eTestingContext struct { + tempDir string + binDir string + homeDir string + configDir string + rcloneConfigDir string + ephemeralRepoDir string +} + +// makeE2eTestingContext sets up a new e2eTestingContext rooted under +// `t.TempDir()`. It creates the skeleton directory structure shown below in the +// temp directory without creating any files. // -// When run on CI, an rclone binary built from HEAD will be on the PATH. When -// running locally, you will likely need to ensure the current binary is on the -// PATH like so: -// -// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/... -// -// In the future, this test will probably be extended to test a number of -// parameters like repo layouts, and runtime may suffer from a combinatorial -// explosion. -func TestEndToEnd(t *testing.T) { +// . +// |-- bin +// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY} +// |-- ephemeralRepo +// `-- user +// `-- .config +// `-- rclone +// `-- rclone.conf +func makeE2eTestingContext(t *testing.T) e2eTestingContext { + tempDir := t.TempDir() + + binDir := filepath.Join(tempDir, "bin") + homeDir := filepath.Join(tempDir, "user") + configDir := filepath.Join(homeDir, ".config") + rcloneConfigDir := filepath.Join(configDir, "rclone") + ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo") + + for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} { + require.NoError(t, os.Mkdir(dir, 0700)) + } + + return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} +} + +// Install the symlink that enables git-annex to invoke "rclone gitannex" +// without explicitly specifying the subcommand. +func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) { + rcloneBinaryPath, err := exec.LookPath("rclone") + require.NoError(t, err) + require.NoError(t, os.Symlink( + rcloneBinaryPath, + filepath.Join(e.binDir, "git-annex-remote-rclone-builtin"))) +} + +// Install a rclone.conf file in an appropriate location in the fake home +// directory. The config defines an rclone remote named "MyRcloneRemote" using +// the local backend. +func (e *e2eTestingContext) installRcloneConfig(t *testing.T) { + // Install the rclone.conf file that defines the remote. + rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf") + rcloneConfigContents := "[MyRcloneRemote]\ntype = local" + require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600)) +} + +// runInRepo runs the given command from within the ephemeral repo directory. To +// prevent accidental changes in the real home directory, it sets the HOME +// variable to a subdirectory of the temp directory. It also ensures that the +// git-annex-remote-rclone-builtin symlink will be found by extending the PATH. +func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) { + fmt.Printf("+ %s %v\n", command, args) + cmd := exec.Command(command, args...) + cmd.Dir = e.ephemeralRepoDir + cmd.Env = []string{ + "HOME=" + e.homeDir, + "PATH=" + os.Getenv("PATH") + ":" + e.binDir, + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Run()) +} + +// createGitRepo creates an empty git repository in the ephemeral repo +// directory. It makes "global" config changes that are ultimately scoped to the +// calling test thanks to runInRepo() overriding the HOME environment variable. +func (e *e2eTestingContext) createGitRepo(t *testing.T) { + e.runInRepo(t, "git", "annex", "version") + e.runInRepo(t, "git", "config", "--global", "user.name", "User Name") + e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com") + e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main") + e.runInRepo(t, "git", "init") + e.runInRepo(t, "git", "annex", "init") +} + +func skipE2eTestIfNecessary(t *testing.T) { if testing.Short() { t.Skip("Skipping due to short mode.") } - // TODO: Support this test on Windows. Need to evaluate the semantics of the + // TODO: Support e2e tests on Windows. Need to evaluate the semantics of the // HOME and PATH environment variables. switch runtime.GOOS { case "darwin", @@ -89,6 +202,23 @@ func TestEndToEnd(t *testing.T) { if _, err := exec.LookPath("git-annex"); err != nil { t.Skipf("Skipping because git-annex was not found: %s", err) } +} + +// This end-to-end test runs `git annex testremote` in a temporary git repo. +// This test will be skipped unless the `rclone` binary on PATH reports the +// expected version. +// +// When run on CI, an rclone binary built from HEAD will be on the PATH. When +// running locally, you will likely need to ensure the current binary is on the +// PATH like so: +// +// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/... +// +// In the future, this test will probably be extended to test a number of +// parameters like repo layouts, and runtime may suffer from a combinatorial +// explosion. +func TestEndToEnd(t *testing.T) { + skipE2eTestIfNecessary(t) // Create a temp directory and chdir there, just in case. originalWd, err := os.Getwd() @@ -97,63 +227,114 @@ func TestEndToEnd(t *testing.T) { require.NoError(t, os.Chdir(tempDir)) defer func() { require.NoError(t, os.Chdir(originalWd)) }() - // Flesh out subdirectories of the temp directory: - // - // . - // |-- bin - // | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY} - // |-- ephemeralRepo - // `-- user - // `-- .config - // `-- rclone - // `-- rclone.conf + testingContext := makeE2eTestingContext(t) + testingContext.installRcloneGitannexSymlink(t) + testingContext.installRcloneConfig(t) + testingContext.createGitRepo(t) - binDir := filepath.Join(tempDir, "bin") - homeDir := filepath.Join(tempDir, "user") - configDir := filepath.Join(homeDir, ".config") - rcloneConfigDir := filepath.Join(configDir, "rclone") - ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo") - for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} { - require.NoError(t, os.Mkdir(dir, 0700)) + testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote", + "type=external", "externaltype=rclone-builtin", "encryption=none", + "rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir) + + testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote") +} + +// For each layout mode, ensure that we're compatible with data written by +// git-annex-remote-rclone. +func TestEndToEndRepoLayoutCompat(t *testing.T) { + skipE2eTestIfNecessary(t) + + if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil { + t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err) } - // Install the symlink that enables git-annex to invoke "rclone gitannex" - // without explicitly specifying the subcommand. - rcloneBinaryPath, err := exec.LookPath("rclone") - require.NoError(t, err) - require.NoError(t, os.Symlink( - rcloneBinaryPath, - filepath.Join(binDir, "git-annex-remote-rclone-builtin"))) + for _, mode := range allLayoutModes() { + mode := mode + t.Run(string(mode), func(t *testing.T) { + t.Parallel() - // Install the rclone.conf file that defines the remote. - rcloneConfigPath := filepath.Join(rcloneConfigDir, "rclone.conf") - rcloneConfigContents := "[MyRcloneRemote]\ntype = local" - require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600)) + // Create a temp directory and chdir there, just in case. + originalWd, err := os.Getwd() + require.NoError(t, err) + tempDir := t.TempDir() + require.NoError(t, os.Chdir(tempDir)) + defer func() { require.NoError(t, os.Chdir(originalWd)) }() - // NOTE: These commands must be run with HOME pointing at an ephemeral - // directory, rather than the real home directory. - cmds := [][]string{ - {"git", "annex", "version"}, - {"git", "config", "--global", "user.name", "User Name"}, - {"git", "config", "--global", "user.email", "user@example.com"}, - {"git", "init"}, - {"git", "annex", "init"}, - {"git", "annex", "initremote", "MyTestRemote", - "type=external", "externaltype=rclone-builtin", "encryption=none", - "rcloneremotename=MyRcloneRemote", "rcloneprefix=" + ephemeralRepoDir}, - {"git", "annex", "testremote", "MyTestRemote"}, - } + tc := makeE2eTestingContext(t) + tc.installRcloneGitannexSymlink(t) + tc.installRcloneConfig(t) + tc.createGitRepo(t) - for _, args := range cmds { - fmt.Printf("+ %v\n", args) - cmd := exec.Command(args[0], args[1:]...) - cmd.Dir = ephemeralRepoDir - cmd.Env = []string{ - "HOME=" + homeDir, - "PATH=" + os.Getenv("PATH") + ":" + binDir, - } - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - require.NoError(t, cmd.Run()) + remoteStorage := filepath.Join(tc.tempDir, "remotePrefix") + require.NoError(t, os.Mkdir(remoteStorage, 0777)) + + tc.runInRepo(t, + "git", "annex", "initremote", "Control", + "type=external", "externaltype=rclone", "encryption=none", + "target=MyRcloneRemote", + "rclone_layout="+string(mode), + "prefix="+remoteStorage) + + tc.runInRepo(t, + "git", "annex", "initremote", "Experiment", + "type=external", "externaltype=rclone-builtin", "encryption=none", + "rcloneremotename=MyRcloneRemote", + "rclonelayout="+string(mode), + "rcloneprefix="+remoteStorage) + + fooFileContents := []byte{1, 2, 3, 4} + fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo") + require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700)) + tc.runInRepo(t, "git", "annex", "add", "foo") + tc.runInRepo(t, "git", "commit", "-m", "Add foo file") + // Git-annex objects are not writable, which prevents `testing` from + // cleaning up the temp directory. We can work around this by + // explicitly dropping any files we add to the annex. + t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") }) + + require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) + require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + // Copy the file to Control and verify it's present on Experiment. + + tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + // Drop the file locally and verify we can copy it back from Experiment. + + tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + // Drop the file from Experiment, copy it back to Experiment, and + // verify it's still present on Control. + + tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo") + require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) + require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo") + require.Equal(t, 1, countFilesRecursively(t, remoteStorage)) + require.True(t, findFileWithContents(t, remoteStorage, fooFileContents)) + + // Drop the file from Control. + + tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo") + require.Equal(t, 0, countFilesRecursively(t, remoteStorage)) + require.False(t, findFileWithContents(t, remoteStorage, fooFileContents)) + }) } }