rclone/cmd/gitannex/e2e_test.go
2024-05-13 18:44:31 +01:00

396 lines
14 KiB
Go

package gitannex
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/stretchr/testify/require"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/buildinfo"
)
// checkRcloneBinaryVersion runs whichever rclone is on the PATH and checks
// whether it reports a version that matches the test's expectations. Returns
// nil when the version is the expected version, otherwise returns an error.
func checkRcloneBinaryVersion(t *testing.T) error {
// versionInfo is a subset of information produced by "core/version".
type versionInfo struct {
Version string
IsGit bool
GoTags string
}
cmd := exec.Command("rclone", "rc", "--loopback", "core/version")
stdout, err := cmd.Output()
if err != nil {
return fmt.Errorf("failed to get rclone version: %w", err)
}
var parsed versionInfo
if err := json.Unmarshal(stdout, &parsed); err != nil {
return fmt.Errorf("failed to parse rclone version: %w", err)
}
if parsed.Version != fs.Version {
return fmt.Errorf("expected version %q, but got %q", fs.Version, parsed.Version)
}
if parsed.IsGit != strings.HasSuffix(fs.Version, "-DEV") {
return errors.New("expected rclone to be a dev build")
}
_, tagString := buildinfo.GetLinkingAndTags()
if parsed.GoTags != tagString {
// TODO: Skip the test when tags do not match.
t.Logf("expected tag string %q, but got %q. Not skipping!", tagString, parsed.GoTags)
}
return nil
}
// countFilesRecursively returns the number of files nested underneath `dir`. It
// counts files only and excludes directories.
func countFilesRecursively(t *testing.T, dir string) int {
remoteFiles, err := os.ReadDir(dir)
require.NoError(t, err)
var count int
for _, f := range remoteFiles {
if f.IsDir() {
subdir := filepath.Join(dir, f.Name())
count += countFilesRecursively(t, subdir)
} else {
count++
}
}
return count
}
func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {
remoteFiles, err := os.ReadDir(dir)
require.NoError(t, err)
for _, f := range remoteFiles {
fPath := filepath.Join(dir, f.Name())
if f.IsDir() {
if findFileWithContents(t, fPath, wantContents) {
return true
}
} else {
contents, err := os.ReadFile(fPath)
require.NoError(t, err)
if bytes.Equal(contents, wantContents) {
return true
}
}
}
return false
}
type e2eTestingContext struct {
tempDir string
binDir string
homeDir string
configDir string
rcloneConfigDir string
ephemeralRepoDir string
}
// makeE2eTestingContext sets up a new e2eTestingContext rooted under
// `t.TempDir()`. It creates the skeleton directory structure shown below in the
// temp directory without creating any files.
//
// .
// |-- bin
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
// |-- ephemeralRepo
// `-- user
// `-- .config
// `-- rclone
// `-- rclone.conf
func makeE2eTestingContext(t *testing.T) e2eTestingContext {
tempDir := t.TempDir()
binDir := filepath.Join(tempDir, "bin")
homeDir := filepath.Join(tempDir, "user")
configDir := filepath.Join(homeDir, ".config")
rcloneConfigDir := filepath.Join(configDir, "rclone")
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
require.NoError(t, os.Mkdir(dir, 0700))
}
return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}
}
// Install the symlink that enables git-annex to invoke "rclone gitannex"
// without explicitly specifying the subcommand.
func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
rcloneBinaryPath, err := exec.LookPath("rclone")
require.NoError(t, err)
require.NoError(t, os.Symlink(
rcloneBinaryPath,
filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
}
// Install a rclone.conf file in an appropriate location in the fake home
// directory. The config defines an rclone remote named "MyRcloneRemote" using
// the local backend.
func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
// Install the rclone.conf file that defines the remote.
rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
}
// runInRepo runs the given command from within the ephemeral repo directory. To
// prevent accidental changes in the real home directory, it sets the HOME
// variable to a subdirectory of the temp directory. It also ensures that the
// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.
func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
fmt.Printf("+ %s %v\n", command, args)
cmd := exec.Command(command, args...)
cmd.Dir = e.ephemeralRepoDir
cmd.Env = []string{
"HOME=" + e.homeDir,
"PATH=" + os.Getenv("PATH") + ":" + e.binDir,
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
require.NoError(t, cmd.Run())
}
// createGitRepo creates an empty git repository in the ephemeral repo
// directory. It makes "global" config changes that are ultimately scoped to the
// calling test thanks to runInRepo() overriding the HOME environment variable.
func (e *e2eTestingContext) createGitRepo(t *testing.T) {
e.runInRepo(t, "git", "annex", "version")
e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com")
e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
e.runInRepo(t, "git", "init")
e.runInRepo(t, "git", "annex", "init")
}
func skipE2eTestIfNecessary(t *testing.T) {
if testing.Short() {
t.Skip("Skipping due to short mode.")
}
// TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
// HOME and PATH environment variables.
switch runtime.GOOS {
case "darwin",
"freebsd",
"linux",
"netbsd",
"openbsd",
"plan9",
"solaris":
default:
t.Skipf("GOOS %q is not supported.", runtime.GOOS)
}
if err := checkRcloneBinaryVersion(t); err != nil {
t.Skipf("Skipping due to rclone version: %s", err)
}
if _, err := exec.LookPath("git-annex"); err != nil {
t.Skipf("Skipping because git-annex was not found: %s", err)
}
}
// This end-to-end test runs `git annex testremote` in a temporary git repo.
// This test will be skipped unless the `rclone` binary on PATH reports the
// expected version.
//
// When run on CI, an rclone binary built from HEAD will be on the PATH. When
// running locally, you will likely need to ensure the current binary is on the
// PATH like so:
//
// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
//
// In the future, this test will probably be extended to test a number of
// parameters like repo layouts, and runtime may suffer from a combinatorial
// explosion.
func TestEndToEnd(t *testing.T) {
skipE2eTestIfNecessary(t)
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
testingContext := makeE2eTestingContext(t)
testingContext.installRcloneGitannexSymlink(t)
testingContext.installRcloneConfig(t)
testingContext.createGitRepo(t)
testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir,
"rclonelayout="+string(mode))
testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
})
}
}
// For each layout mode, migrate a single remote from git-annex-remote-rclone to
// git-annex-remote-rclone-builtin and run `git annex testremote`.
func TestEndToEndMigration(t *testing.T) {
skipE2eTestIfNecessary(t)
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
tc := makeE2eTestingContext(t)
tc.installRcloneGitannexSymlink(t)
tc.installRcloneConfig(t)
tc.createGitRepo(t)
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
require.NoError(t, os.Mkdir(remoteStorage, 0777))
tc.runInRepo(t,
"git", "annex", "initremote", "MigratedRemote",
"type=external", "externaltype=rclone", "encryption=none",
"target=MyRcloneRemote",
"rclone_layout="+string(mode),
"prefix="+remoteStorage,
)
fooFileContents := []byte{1, 2, 3, 4}
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
tc.runInRepo(t, "git", "annex", "add", "foo")
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
// Git-annex objects are not writable, which prevents `testing` from
// cleaning up the temp directory. We can work around this by
// explicitly dropping any files we add to the annex.
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
tc.runInRepo(t, "git", "annex", "copy", "--to=MigratedRemote", "foo")
tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo")
tc.runInRepo(t,
"git", "annex", "enableremote", "MigratedRemote",
"externaltype=rclone-builtin",
"rcloneremotename=MyRcloneRemote",
"rclonelayout="+string(mode),
"rcloneprefix="+remoteStorage,
)
tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo")
tc.runInRepo(t, "git", "annex", "testremote", "MigratedRemote")
})
}
}
// For each layout mode, create two git-annex remotes with externaltype=rclone
// and externaltype=rclone-builtin respectively. Test that files copied to one
// remote are present on the other. Similarly, test that files deleted from one
// are removed on the other.
func TestEndToEndRepoLayoutCompat(t *testing.T) {
skipE2eTestIfNecessary(t)
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
tc := makeE2eTestingContext(t)
tc.installRcloneGitannexSymlink(t)
tc.installRcloneConfig(t)
tc.createGitRepo(t)
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
require.NoError(t, os.Mkdir(remoteStorage, 0777))
tc.runInRepo(t,
"git", "annex", "initremote", "Control",
"type=external", "externaltype=rclone", "encryption=none",
"target=MyRcloneRemote",
"rclone_layout="+string(mode),
"prefix="+remoteStorage)
tc.runInRepo(t,
"git", "annex", "initremote", "Experiment",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote",
"rclonelayout="+string(mode),
"rcloneprefix="+remoteStorage)
fooFileContents := []byte{1, 2, 3, 4}
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
tc.runInRepo(t, "git", "annex", "add", "foo")
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
// Git-annex objects are not writable, which prevents `testing` from
// cleaning up the temp directory. We can work around this by
// explicitly dropping any files we add to the annex.
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Copy the file to Control and verify it's present on Experiment.
tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file locally and verify we can copy it back from Experiment.
tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file from Experiment, copy it back to Experiment, and
// verify it's still present on Control.
tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file from Control.
tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
})
}
}