Enhancement: option to send HTTP over unix socket

add tests for unix socket connection

switch HTTP rest-server test to use any free port

allow rest-server test graceful shutdown opportunity

doc/030_preparing_a_new_repo.rst

@@ -201,15 +201,16 @@ scheme like this:
     $ restic -r rest:http://host:8000/ init
 
 Depending on your REST server setup, you can use HTTPS protocol,
-password protection, multiple repositories or any combination of
+unix socket, password protection, multiple repositories or any
-those features. The TCP/IP port is also configurable. Here
+combination of those features. The TCP/IP port is also configurable.
-are some more examples:
+Here are some more examples:
 
 .. code-block:: console
 
     $ restic -r rest:https://host:8000/ init
     $ restic -r rest:https://user:pass@host:8000/ init
     $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init
+    $ restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ init
 
 The server username and password can be specified using environment
 variables as well:

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/minio/sha256-simd v1.0.1
 	github.com/ncw/swift/v2 v2.0.2
+	github.com/peterbourgon/unixtransport v0.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
 	github.com/pkg/sftp v1.13.6

go.sum

@@ -128,6 +128,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
@@ -141,6 +142,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
 github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
+github.com/peterbourgon/unixtransport v0.0.4 h1:UTF0FxXCAglvoZz9jaGPYjEg52DjBLDYGMJvJni6Tfw=
+github.com/peterbourgon/unixtransport v0.0.4/go.mod h1:o8aUkOCa8W/BIXpi15uKvbSabjtBh0JhSOJGSfoOhAU=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -210,6 +216,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -221,6 +228,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -231,6 +239,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -246,12 +255,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -271,6 +282,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -315,6 +327,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/backend/http_transport.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/peterbourgon/unixtransport"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 )
@@ -82,6 +83,8 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
 		TLSClientConfig:       &tls.Config{},
 	}
 
+	unixtransport.Register(tr)
+
 	if opts.InsecureTLS {
 		tr.TLSClientConfig.InsecureSkipVerify = true
 	}

internal/backend/rest/config_test.go

@@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{
 			Connections: 5,
 		},
 	},
+	{
+		S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/",
+		Cfg: Config{
+			URL:         parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"),
+			Connections: 5,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {

internal/backend/rest/rest_test.go

@@ -1,11 +1,18 @@
+//go:build go1.20
+// +build go1.20
+
 package rest_test
 
 import (
+	"bufio"
 	"context"
-	"net"
+	"fmt"
 	"net/url"
 	"os"
 	"os/exec"
+	"regexp"
+	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -14,54 +21,133 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) {
+var (
+	serverStartedRE = regexp.MustCompile("^start server on (.*)$")
+)
+
+func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) {
 	srv, err := exec.LookPath("rest-server")
 	if err != nil {
 		t.Skip(err)
 	}
 
-	cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir)
+	// create our own context, so that our cleanup can cancel and wait for completion
+	// this will ensure any open ports, open unix sockets etc are properly closed
+	processCtx, cancel := context.WithCancel(ctx)
+	cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr)
+
+	// this cancel func is called by when the process context is done
+	cmd.Cancel = func() error {
+		// we execute in a Go-routine as we know the caller will
+		// be waiting on a .Wait() regardless
+		go func() {
+			// try to send a graceful termination signal
+			if cmd.Process.Signal(syscall.SIGTERM) == nil {
+				// if we succeed, then wait a few seconds
+				time.Sleep(2 * time.Second)
+			}
+			// and then make sure it's killed either way, ignoring any error code
+			_ = cmd.Process.Kill()
+		}()
+		return nil
+	}
+
+	// this is the cleanup function that we return the caller,
+	// which will cancel our process context, and then wait for it to finish
+	cleanup := func() {
+		cancel()
+		_ = cmd.Wait()
+	}
+
+	// but in-case we don't finish this method, e.g. by calling t.Fatal()
+	// we also defer a call to clean it up ourselves, guarded by a flag to
+	// indicate that we returned the function to the caller to deal with.
+	callerWillCleanUp := false
+	defer func() {
+		if !callerWillCleanUp {
+			cleanup()
+		}
+	}()
+
+	// send stdout to our std out
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stdout
+
+	// capture stderr with a pipe, as we want to examine this output
+	// to determine when the server is started and listening.
+	cmdErr, err := cmd.StderrPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// start the rest-server
 	if err := cmd.Start(); err != nil {
 		t.Fatal(err)
 	}
 
-	// wait until the TCP port is reachable
+	// create a channel to receive the actual listen address on
-	var success bool
+	listenAddrCh := make(chan string)
-	for i := 0; i < 10; i++ {
+	go func() {
-		time.Sleep(200 * time.Millisecond)
+		defer close(listenAddrCh)
-
+		matched := false
-		c, err := net.Dial("tcp", "localhost:8000")
+		br := bufio.NewReader(cmdErr)
+		for {
+			line, err := br.ReadString('\n')
 			if err != nil {
-			continue
+				// we ignore errors, as code that relies on this
+				// will happily fail via timeout and empty closed
+				// channel.
+				return
 			}
 
-		success = true
+			line = strings.Trim(line, "\r\n")
-		if err := c.Close(); err != nil {
+			if !matched {
-			t.Fatal(err)
+				// look for the server started message, and return the address
+				// that it's listening on
+				matchedServerListen := serverStartedRE.FindSubmatch([]byte(line))
+				if len(matchedServerListen) == 2 {
+					listenAddrCh <- string(matchedServerListen[1])
+					matched = true
 				}
 			}
+			fmt.Fprintln(os.Stdout, line) // print all output to console
+		}
+	}()
+
+	// wait for us to get an address,
+	// or the parent context to cancel,
+	// or for us to timeout
+	var actualListenAddr string
+	select {
+	case <-processCtx.Done():
+		t.Fatal(context.Canceled)
+	case <-time.NewTimer(2 * time.Second).C:
+		t.Fatal(context.DeadlineExceeded)
+	case a, ok := <-listenAddrCh:
+		if !ok {
+			t.Fatal(context.Canceled)
+		}
+		actualListenAddr = a
+	}
 
-	if !success {
+	// this translate the address that the server is listening on
-		t.Fatal("unable to connect to rest server")
+	// to a URL suitable for us to connect to
-		return nil, nil
+	var addrToConnectTo string
+	if strings.HasPrefix(reqListenAddr, "unix:") {
+		addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr)
+	} else {
+		// while we may listen on 0.0.0.0, we connect to localhost
+		addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1))
 	}
 
-	url, err := url.Parse("http://localhost:8000/restic-test/")
+	// parse to a URL
+	url, err := url.Parse(addrToConnectTo)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	cleanup := func() {
+	// indicate that we've completed successfully, and that the caller
-		if err := cmd.Process.Kill(); err != nil {
+	// is responsible for calling cleanup
-			t.Fatal(err)
+	callerWillCleanUp = true
-		}
-
-		// ignore errors, we've killed the process
-		_ = cmd.Wait()
-	}
-
 	return url, cleanup
 }
 
@@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunTests(t)
@@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunBenchmarks(t)

internal/backend/rest/rest_unix_test.go

@@ -0,0 +1,30 @@
+//go:build !windows && go1.20
+// +build !windows,go1.20
+
+package rest_test
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestBackendRESTWithUnixSocket(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir := rtest.TempDir(t)
+	serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock")))
+	defer cleanup()
+
+	newTestSuite(serverURL, false).RunTests(t)
+}