Merge pull request #4655 from ae-govau/unixsocket

Enhancement: option to send HTTP over unix socket
This commit is contained in:
Michael Eischer 2024-04-03 19:29:21 +00:00 committed by GitHub
commit 6cca1d5705
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 198 additions and 43 deletions

View file

@ -74,7 +74,7 @@ jobs:
- name: Get programs (Linux/macOS)
run: |
echo "build Go tools"
go install github.com/restic/rest-server/cmd/rest-server@latest
go install github.com/restic/rest-server/cmd/rest-server@master
echo "install minio server"
mkdir $HOME/bin
@ -106,7 +106,7 @@ jobs:
$ProgressPreference = 'SilentlyContinue'
echo "build Go tools"
go install github.com/restic/rest-server/...
go install github.com/restic/rest-server/cmd/rest-server@master
echo "install minio server"
mkdir $Env:USERPROFILE/bin

View file

@ -0,0 +1,14 @@
Enhancement: support connection to rest-server using unix socket
Restic now supports connecting to rest-server using a unix socket for
rest-server version 0.13.0 or later.
This allows running restic as follows:
```
rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
```
https://github.com/restic/restic/issues/4287
https://github.com/restic/restic/pull/4655

View file

@ -201,15 +201,16 @@ scheme like this:
$ restic -r rest:http://host:8000/ init
Depending on your REST server setup, you can use HTTPS protocol,
password protection, multiple repositories or any combination of
those features. The TCP/IP port is also configurable. Here
are some more examples:
unix socket, password protection, multiple repositories or any
combination of those features. The TCP/IP port is also configurable.
Here are some more examples:
.. code-block:: console
$ restic -r rest:https://host:8000/ init
$ restic -r rest:https://user:pass@host:8000/ init
$ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init
$ restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ init
The server username and password can be specified using environment
variables as well:

1
go.mod
View file

@ -17,6 +17,7 @@ require (
github.com/minio/minio-go/v7 v7.0.66
github.com/minio/sha256-simd v1.0.1
github.com/ncw/swift/v2 v2.0.2
github.com/peterbourgon/unixtransport v0.0.4
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.6

13
go.sum
View file

@ -128,6 +128,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
@ -141,6 +142,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
github.com/peterbourgon/unixtransport v0.0.4 h1:UTF0FxXCAglvoZz9jaGPYjEg52DjBLDYGMJvJni6Tfw=
github.com/peterbourgon/unixtransport v0.0.4/go.mod h1:o8aUkOCa8W/BIXpi15uKvbSabjtBh0JhSOJGSfoOhAU=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -210,6 +216,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -221,6 +228,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -231,6 +239,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -246,12 +255,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -271,6 +282,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -315,6 +327,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -10,6 +10,7 @@ import (
"strings"
"time"
"github.com/peterbourgon/unixtransport"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
)
@ -82,6 +83,8 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
TLSClientConfig: &tls.Config{},
}
unixtransport.Register(tr)
if opts.InsecureTLS {
tr.TLSClientConfig.InsecureSkipVerify = true
}

View file

@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{
Connections: 5,
},
},
{
S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/",
Cfg: Config{
URL: parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"),
Connections: 5,
},
},
}
func TestParseConfig(t *testing.T) {

View file

@ -1,11 +1,18 @@
//go:build go1.20
// +build go1.20
package rest_test
import (
"bufio"
"context"
"net"
"fmt"
"net/url"
"os"
"os/exec"
"regexp"
"strings"
"syscall"
"testing"
"time"
@ -14,54 +21,133 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) {
var (
serverStartedRE = regexp.MustCompile("^start server on (.*)$")
)
func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) {
srv, err := exec.LookPath("rest-server")
if err != nil {
t.Skip(err)
}
cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir)
// create our own context, so that our cleanup can cancel and wait for completion
// this will ensure any open ports, open unix sockets etc are properly closed
processCtx, cancel := context.WithCancel(ctx)
cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr)
// this cancel func is called by when the process context is done
cmd.Cancel = func() error {
// we execute in a Go-routine as we know the caller will
// be waiting on a .Wait() regardless
go func() {
// try to send a graceful termination signal
if cmd.Process.Signal(syscall.SIGTERM) == nil {
// if we succeed, then wait a few seconds
time.Sleep(2 * time.Second)
}
// and then make sure it's killed either way, ignoring any error code
_ = cmd.Process.Kill()
}()
return nil
}
// this is the cleanup function that we return the caller,
// which will cancel our process context, and then wait for it to finish
cleanup := func() {
cancel()
_ = cmd.Wait()
}
// but in-case we don't finish this method, e.g. by calling t.Fatal()
// we also defer a call to clean it up ourselves, guarded by a flag to
// indicate that we returned the function to the caller to deal with.
callerWillCleanUp := false
defer func() {
if !callerWillCleanUp {
cleanup()
}
}()
// send stdout to our std out
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stdout
// capture stderr with a pipe, as we want to examine this output
// to determine when the server is started and listening.
cmdErr, err := cmd.StderrPipe()
if err != nil {
t.Fatal(err)
}
// start the rest-server
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
// wait until the TCP port is reachable
var success bool
for i := 0; i < 10; i++ {
time.Sleep(200 * time.Millisecond)
c, err := net.Dial("tcp", "localhost:8000")
// create a channel to receive the actual listen address on
listenAddrCh := make(chan string)
go func() {
defer close(listenAddrCh)
matched := false
br := bufio.NewReader(cmdErr)
for {
line, err := br.ReadString('\n')
if err != nil {
continue
// we ignore errors, as code that relies on this
// will happily fail via timeout and empty closed
// channel.
return
}
success = true
if err := c.Close(); err != nil {
t.Fatal(err)
line = strings.Trim(line, "\r\n")
if !matched {
// look for the server started message, and return the address
// that it's listening on
matchedServerListen := serverStartedRE.FindSubmatch([]byte(line))
if len(matchedServerListen) == 2 {
listenAddrCh <- string(matchedServerListen[1])
matched = true
}
}
fmt.Fprintln(os.Stdout, line) // print all output to console
}
}()
// wait for us to get an address,
// or the parent context to cancel,
// or for us to timeout
var actualListenAddr string
select {
case <-processCtx.Done():
t.Fatal(context.Canceled)
case <-time.NewTimer(2 * time.Second).C:
t.Fatal(context.DeadlineExceeded)
case a, ok := <-listenAddrCh:
if !ok {
t.Fatal(context.Canceled)
}
actualListenAddr = a
}
if !success {
t.Fatal("unable to connect to rest server")
return nil, nil
// this translate the address that the server is listening on
// to a URL suitable for us to connect to
var addrToConnectTo string
if strings.HasPrefix(reqListenAddr, "unix:") {
addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr)
} else {
// while we may listen on 0.0.0.0, we connect to localhost
addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1))
}
url, err := url.Parse("http://localhost:8000/restic-test/")
// parse to a URL
url, err := url.Parse(addrToConnectTo)
if err != nil {
t.Fatal(err)
}
cleanup := func() {
if err := cmd.Process.Kill(); err != nil {
t.Fatal(err)
}
// ignore errors, we've killed the process
_ = cmd.Wait()
}
// indicate that we've completed successfully, and that the caller
// is responsible for calling cleanup
callerWillCleanUp = true
return url, cleanup
}
@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) {
defer cancel()
dir := rtest.TempDir(t)
serverURL, cleanup := runRESTServer(ctx, t, dir)
serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
defer cleanup()
newTestSuite(serverURL, false).RunTests(t)
@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) {
defer cancel()
dir := rtest.TempDir(t)
serverURL, cleanup := runRESTServer(ctx, t, dir)
serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
defer cleanup()
newTestSuite(serverURL, false).RunBenchmarks(t)

View file

@ -0,0 +1,30 @@
//go:build !windows && go1.20
// +build !windows,go1.20
package rest_test
import (
"context"
"fmt"
"path"
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestBackendRESTWithUnixSocket(t *testing.T) {
defer func() {
if t.Skipped() {
rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
}
}()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := rtest.TempDir(t)
serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock")))
defer cleanup()
newTestSuite(serverURL, false).RunTests(t)
}