Adding S3 support for HTTP domain validation (#1970)

Co-authored-by: Fernandez Ludovic <ldez@users.noreply.github.com>

.golangci.yml

@@ -181,6 +181,8 @@ issues:
       text: load is a global variable
     - path: 'providers/dns/([\d\w]+/)*[\d\w]+_test.go'
       text: 'envTest is a global variable'
+    - path: 'providers/http/([\d\w]+/)*[\d\w]+_test.go'
+      text: 'envTest is a global variable'
     - path: providers/dns/namecheap/namecheap_test.go
       text: 'testCases is a global variable'
     - path: providers/dns/acmedns/acmedns_test.go
@@ -222,4 +224,3 @@ issues:
       text: 'Duplicate words \(0\) found'
     - path: cmd/cmd_renew.go
       text: 'cyclomatic complexity 16 of func `renewForDomains` is high'
-

cmd/flags.go

@@ -87,6 +87,10 @@ func CreateFlags(defaultPath string) []cli.Flag {
 			Name:  "http.memcached-host",
 			Usage: "Set the memcached host(s) to use for HTTP-01 based challenges. Challenges will be written to all specified hosts.",
 		},
+		&cli.StringFlag{
+			Name:  "http.s3-bucket",
+			Usage: "Set the S3 bucket name to use for HTTP-01 based challenges. Challenges will be written to the S3 bucket.",
+		},
 		&cli.BoolFlag{
 			Name:  "tls",
 			Usage: "Use the TLS-ALPN-01 challenge to solve challenges. Can be mixed with other types of challenges.",

cmd/setup_challenges.go

@@ -13,6 +13,7 @@ import (
 	"github.com/go-acme/lego/v4/log"
 	"github.com/go-acme/lego/v4/providers/dns"
 	"github.com/go-acme/lego/v4/providers/http/memcached"
+	"github.com/go-acme/lego/v4/providers/http/s3"
 	"github.com/go-acme/lego/v4/providers/http/webroot"
 	"github.com/urfave/cli/v2"
 )
@@ -41,6 +42,7 @@ func setupChallenges(ctx *cli.Context, client *lego.Client) {
 	}
 }
 
+//nolint:gocyclo // the complexity is expected.
 func setupHTTPProvider(ctx *cli.Context) challenge.Provider {
 	switch {
 	case ctx.IsSet("http.webroot"):
@@ -55,6 +57,12 @@ func setupHTTPProvider(ctx *cli.Context) challenge.Provider {
 			log.Fatal(err)
 		}
 		return ps
+	case ctx.IsSet("http.s3-bucket"):
+		ps, err := s3.NewHTTPProvider(ctx.String("http.s3-bucket"))
+		if err != nil {
+			log.Fatal(err)
+		}
+		return ps
 	case ctx.IsSet("http.port"):
 		iface := ctx.String("http.port")
 		if !strings.Contains(iface, ":") {

docs/data/zz_cli_help.toml

@@ -35,6 +35,7 @@ GLOBAL OPTIONS:
    --http.proxy-header value                                    Validate against this HTTP header when solving HTTP-01 based challenges behind a reverse proxy. (default: "Host")
    --http.webroot value                                         Set the webroot folder to use for HTTP-01 based challenges to write directly to the .well-known/acme-challenge file. This disables the built-in server and expects the given directory to be publicly served with access to .well-known/acme-challenge
    --http.memcached-host value [ --http.memcached-host value ]  Set the memcached host(s) to use for HTTP-01 based challenges. Challenges will be written to all specified hosts.
+   --http.s3-bucket value                                       Set the S3 bucket name to use for HTTP-01 based challenges. Challenges will be written to the S3 bucket.
    --tls                                                        Use the TLS-ALPN-01 challenge to solve challenges. Can be mixed with other types of challenges. (default: false)
    --tls.port value                                             Set the port and interface to use for TLS-ALPN-01 based challenges to listen on. Supported: interface:port or :port. (default: ":443")
    --dns value                                                  Solve a DNS-01 challenge using the specified provider. Can be mixed with other types of challenges. Run 'lego dnshelp' for help on usage.

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.27
 	github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2
 	github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.19.3
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/civo/civogo v0.3.11
@@ -95,11 +96,16 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect

go.sum

@@ -76,6 +76,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k=
 github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
 github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw=
 github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA=
@@ -88,12 +90,22 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w=
 github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2 h1:PwNeYoonBzmTdCztKiiutws3U24KrnDBuabzRfIlZY4=
 github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2/go.mod h1:gQhLZrTEath4zik5ixIe6axvgY5jJrgSBDJ360Fxnco=
 github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4 h1:p4mTxJfCAyiTT4Wp6p/mOPa6j5MqCSRGot8qZwFs+Z0=
 github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4/go.mod h1:VBLWpaHvhQNeu7N9rMEf00SWeOONb/HvaDUxe/7b44k=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4=

providers/http/s3/s3.go

@@ -0,0 +1,77 @@
+// Package s3 implements a HTTP provider for solving the HTTP-01 challenge using web server's root path.
+package s3
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/go-acme/lego/v4/challenge/http01"
+)
+
+// HTTPProvider implements ChallengeProvider for `http-01` challenge.
+type HTTPProvider struct {
+	bucket string
+	client *s3.Client
+}
+
+// NewHTTPProvider returns a HTTPProvider instance with a configured s3 bucket and aws session.
+// Credentials must be passed in the environment variables.
+func NewHTTPProvider(bucket string) (*HTTPProvider, error) {
+	if bucket == "" {
+		return nil, fmt.Errorf("s3: bucket name missing")
+	}
+
+	ctx := context.Background()
+
+	cfg, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("s3: unable to create AWS config: %w", err)
+	}
+
+	client := s3.NewFromConfig(cfg)
+
+	return &HTTPProvider{
+		bucket: bucket,
+		client: client,
+	}, nil
+}
+
+// Present makes the token available at `HTTP01ChallengePath(token)` by creating a file in the given s3 bucket.
+func (s *HTTPProvider) Present(domain, token, keyAuth string) error {
+	ctx := context.Background()
+
+	params := &s3.PutObjectInput{
+		ACL:    "public-read",
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(strings.Trim(http01.ChallengePath(token), "/")),
+		Body:   bytes.NewReader([]byte(keyAuth)),
+	}
+
+	_, err := s.client.PutObject(ctx, params)
+	if err != nil {
+		return fmt.Errorf("s3: failed to upload token to s3: %w", err)
+	}
+	return nil
+}
+
+// CleanUp removes the file created for the challenge.
+func (s *HTTPProvider) CleanUp(domain, token, keyAuth string) error {
+	ctx := context.Background()
+
+	params := &s3.DeleteObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(strings.Trim(http01.ChallengePath(token), "/")),
+	}
+
+	_, err := s.client.DeleteObject(ctx, params)
+	if err != nil {
+		return fmt.Errorf("s3: could not remove file in s3 bucket after HTTP challenge: %w", err)
+	}
+
+	return nil
+}

providers/http/s3/s3.toml

@@ -0,0 +1,54 @@
+Name = "Amazon S3"
+Description = ''''''
+URL = "https://aws.amazon.com/s3/"
+Code = "s3"
+Since = "v4.14.0"
+
+Example = '''
+AWS_ACCESS_KEY_ID=your_key_id \
+AWS_SECRET_ACCESS_KEY=your_secret_access_key \
+AWS_REGION=aws-region \
+lego --domains example.com --email your_example@email.com --http --http.s3-bucket your_s3_bucket --accept-tos=true run
+'''
+
+Additional = '''
+## Description
+
+AWS Credentials are automatically detected in the following locations and prioritized in the following order:
+
+1. Environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, [`AWS_SESSION_TOKEN`]
+2. Shared credentials file (defaults to `~/.aws/credentials`, profiles can be specified using `AWS_PROFILE`)
+3. Amazon EC2 IAM role
+
+The AWS Region is automatically detected in the following locations and prioritized in the following order:
+
+1. Environment variables: `AWS_REGION`
+2. Shared configuration file if `AWS_SDK_LOAD_CONFIG` is set (defaults to `~/.aws/config`, profiles can be specified using `AWS_PROFILE`)
+
+See also: https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/
+
+### Broad privileges for testing purposes
+
+Will need to create an S3 bucket which has read permissions set for Everyone (public access).
+The S3 bucket doesn't require static website hosting to be enabled.
+AWS_REGION must match the region where the s3 bucket is hosted.
+'''
+
+[Configuration]
+  [Configuration.Credentials]
+    AWS_ACCESS_KEY_ID = "Managed by the AWS client. Access key ID (`AWS_ACCESS_KEY_ID_FILE` is not supported, use `AWS_SHARED_CREDENTIALS_FILE` instead)"
+    AWS_SECRET_ACCESS_KEY = "Managed by the AWS client. Secret access key (`AWS_SECRET_ACCESS_KEY_FILE` is not supported, use `AWS_SHARED_CREDENTIALS_FILE` instead)"
+    AWS_REGION = "Managed by the AWS client (`AWS_REGION_FILE` is not supported)"
+    S3_BUCKET = "Name of the s3 bucket"
+    AWS_PROFILE = "Managed by the AWS client (`AWS_PROFILE_FILE` is not supported)"
+    AWS_SDK_LOAD_CONFIG = "Managed by the AWS client. Retrieve the region from the CLI config file (`AWS_SDK_LOAD_CONFIG_FILE` is not supported)"
+    AWS_ASSUME_ROLE_ARN = "Managed by the AWS Role ARN (`AWS_ASSUME_ROLE_ARN_FILE` is not supported)"
+    AWS_EXTERNAL_ID = "Managed by STS AssumeRole API operation (`AWS_EXTERNAL_ID_FILE` is not supported)"
+  [Configuration.Additional]
+    AWS_SHARED_CREDENTIALS_FILE = "Managed by the AWS client. Shared credentials file."
+    AWS_MAX_RETRIES = "The number of maximum returns the service will use to make an individual API request"
+
+[Links]
+  API = "https://docs.aws.amazon.com/AmazonS3/latest/userguide//Welcome.html"
+  GoClient = "https://docs.aws.amazon.com/sdk-for-go/"
+

providers/http/s3/s3_test.go

@@ -0,0 +1,80 @@
+// Package s3 implements a HTTP provider for solving the HTTP-01 challenge
+// using AWS S3 in combination with AWS CloudFront.
+package s3
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/go-acme/lego/v4/challenge/http01"
+	"github.com/go-acme/lego/v4/platform/tester"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	domain  = "example.com"
+	token   = "foo"
+	keyAuth = "bar"
+)
+
+var envTest = tester.NewEnvTest(
+	"AWS_ACCESS_KEY_ID",
+	"AWS_SECRET_ACCESS_KEY",
+	"AWS_REGION",
+	"S3_BUCKET")
+
+func TestLiveNewHTTPProvider_Valid(t *testing.T) {
+	if !envTest.IsLiveTest() {
+		t.Skip("skipping live test")
+	}
+
+	envTest.RestoreEnv()
+
+	_, err := NewHTTPProvider(envTest.GetValue("S3_BUCKET"))
+	require.NoError(t, err)
+}
+
+func TestLiveNewHTTPProvider(t *testing.T) {
+	if !envTest.IsLiveTest() {
+		t.Skip("skipping live test")
+	}
+
+	envTest.RestoreEnv()
+
+	s3Bucket := os.Getenv("S3_BUCKET")
+
+	provider, err := NewHTTPProvider(s3Bucket)
+	require.NoError(t, err)
+
+	// Present
+
+	err = provider.Present(domain, token, keyAuth)
+	require.NoError(t, err)
+
+	chlgPath := fmt.Sprintf("http://%s.s3.%s.amazonaws.com%s",
+		s3Bucket, envTest.GetValue("AWS_REGION"), http01.ChallengePath(token))
+
+	resp, err := http.Get(chlgPath)
+	require.NoError(t, err)
+
+	defer func() { _ = resp.Body.Close() }()
+
+	data, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	assert.Equal(t, []byte(keyAuth), data)
+
+	// CleanUp
+
+	err = provider.CleanUp(domain, token, keyAuth)
+	require.NoError(t, err)
+
+	cleanupResp, err := http.Get(chlgPath)
+	require.NoError(t, err)
+
+	assert.Equal(t, cleanupResp.StatusCode, 403)
+}