From 6c13564bad925c543b87ed96f3f6b33518a04817 Mon Sep 17 00:00:00 2001 From: kingcdavid Date: Thu, 27 Jul 2023 11:42:48 +0100 Subject: [PATCH] Adding S3 support for HTTP domain validation (#1970) Co-authored-by: Fernandez Ludovic --- .golangci.yml | 3 +- cmd/flags.go | 4 ++ cmd/setup_challenges.go | 8 ++++ docs/data/zz_cli_help.toml | 1 + go.mod | 6 +++ go.sum | 12 ++++++ providers/http/s3/s3.go | 77 ++++++++++++++++++++++++++++++++++ providers/http/s3/s3.toml | 54 ++++++++++++++++++++++++ providers/http/s3/s3_test.go | 80 ++++++++++++++++++++++++++++++++++++ 9 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 providers/http/s3/s3.go create mode 100644 providers/http/s3/s3.toml create mode 100644 providers/http/s3/s3_test.go diff --git a/.golangci.yml b/.golangci.yml index 561ec901..b84462b4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -181,6 +181,8 @@ issues: text: load is a global variable - path: 'providers/dns/([\d\w]+/)*[\d\w]+_test.go' text: 'envTest is a global variable' + - path: 'providers/http/([\d\w]+/)*[\d\w]+_test.go' + text: 'envTest is a global variable' - path: providers/dns/namecheap/namecheap_test.go text: 'testCases is a global variable' - path: providers/dns/acmedns/acmedns_test.go @@ -222,4 +224,3 @@ issues: text: 'Duplicate words \(0\) found' - path: cmd/cmd_renew.go text: 'cyclomatic complexity 16 of func `renewForDomains` is high' - diff --git a/cmd/flags.go b/cmd/flags.go index 4a516ab4..cc9c1edf 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -87,6 +87,10 @@ func CreateFlags(defaultPath string) []cli.Flag { Name: "http.memcached-host", Usage: "Set the memcached host(s) to use for HTTP-01 based challenges. Challenges will be written to all specified hosts.", }, + &cli.StringFlag{ + Name: "http.s3-bucket", + Usage: "Set the S3 bucket name to use for HTTP-01 based challenges. Challenges will be written to the S3 bucket.", + }, &cli.BoolFlag{ Name: "tls", Usage: "Use the TLS-ALPN-01 challenge to solve challenges. Can be mixed with other types of challenges.", diff --git a/cmd/setup_challenges.go b/cmd/setup_challenges.go index 938ee745..719f8dd6 100644 --- a/cmd/setup_challenges.go +++ b/cmd/setup_challenges.go @@ -13,6 +13,7 @@ import ( "github.com/go-acme/lego/v4/log" "github.com/go-acme/lego/v4/providers/dns" "github.com/go-acme/lego/v4/providers/http/memcached" + "github.com/go-acme/lego/v4/providers/http/s3" "github.com/go-acme/lego/v4/providers/http/webroot" "github.com/urfave/cli/v2" ) @@ -41,6 +42,7 @@ func setupChallenges(ctx *cli.Context, client *lego.Client) { } } +//nolint:gocyclo // the complexity is expected. func setupHTTPProvider(ctx *cli.Context) challenge.Provider { switch { case ctx.IsSet("http.webroot"): @@ -55,6 +57,12 @@ func setupHTTPProvider(ctx *cli.Context) challenge.Provider { log.Fatal(err) } return ps + case ctx.IsSet("http.s3-bucket"): + ps, err := s3.NewHTTPProvider(ctx.String("http.s3-bucket")) + if err != nil { + log.Fatal(err) + } + return ps case ctx.IsSet("http.port"): iface := ctx.String("http.port") if !strings.Contains(iface, ":") { diff --git a/docs/data/zz_cli_help.toml b/docs/data/zz_cli_help.toml index 27296037..54e848cc 100644 --- a/docs/data/zz_cli_help.toml +++ b/docs/data/zz_cli_help.toml @@ -35,6 +35,7 @@ GLOBAL OPTIONS: --http.proxy-header value Validate against this HTTP header when solving HTTP-01 based challenges behind a reverse proxy. (default: "Host") --http.webroot value Set the webroot folder to use for HTTP-01 based challenges to write directly to the .well-known/acme-challenge file. This disables the built-in server and expects the given directory to be publicly served with access to .well-known/acme-challenge --http.memcached-host value [ --http.memcached-host value ] Set the memcached host(s) to use for HTTP-01 based challenges. Challenges will be written to all specified hosts. + --http.s3-bucket value Set the S3 bucket name to use for HTTP-01 based challenges. Challenges will be written to the S3 bucket. --tls Use the TLS-ALPN-01 challenge to solve challenges. Can be mixed with other types of challenges. (default: false) --tls.port value Set the port and interface to use for TLS-ALPN-01 based challenges to listen on. Supported: interface:port or :port. (default: ":443") --dns value Solve a DNS-01 challenge using the specified provider. Can be mixed with other types of challenges. Run 'lego dnshelp' for help on usage. diff --git a/go.mod b/go.mod index dd44dc41..28db069a 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.27 github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2 github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 github.com/cenkalti/backoff/v4 v4.2.1 github.com/civo/civogo v0.3.11 @@ -95,11 +96,16 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect github.com/aws/smithy-go v1.13.5 // indirect diff --git a/go.sum b/go.sum index 1b701724..dd59e505 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= @@ -88,12 +90,22 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w= github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2 h1:PwNeYoonBzmTdCztKiiutws3U24KrnDBuabzRfIlZY4= github.com/aws/aws-sdk-go-v2/service/lightsail v1.27.2/go.mod h1:gQhLZrTEath4zik5ixIe6axvgY5jJrgSBDJ360Fxnco= github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4 h1:p4mTxJfCAyiTT4Wp6p/mOPa6j5MqCSRGot8qZwFs+Z0= github.com/aws/aws-sdk-go-v2/service/route53 v1.28.4/go.mod h1:VBLWpaHvhQNeu7N9rMEf00SWeOONb/HvaDUxe/7b44k= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A= github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= diff --git a/providers/http/s3/s3.go b/providers/http/s3/s3.go new file mode 100644 index 00000000..0df63bf1 --- /dev/null +++ b/providers/http/s3/s3.go @@ -0,0 +1,77 @@ +// Package s3 implements a HTTP provider for solving the HTTP-01 challenge using web server's root path. +package s3 + +import ( + "bytes" + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/go-acme/lego/v4/challenge/http01" +) + +// HTTPProvider implements ChallengeProvider for `http-01` challenge. +type HTTPProvider struct { + bucket string + client *s3.Client +} + +// NewHTTPProvider returns a HTTPProvider instance with a configured s3 bucket and aws session. +// Credentials must be passed in the environment variables. +func NewHTTPProvider(bucket string) (*HTTPProvider, error) { + if bucket == "" { + return nil, fmt.Errorf("s3: bucket name missing") + } + + ctx := context.Background() + + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("s3: unable to create AWS config: %w", err) + } + + client := s3.NewFromConfig(cfg) + + return &HTTPProvider{ + bucket: bucket, + client: client, + }, nil +} + +// Present makes the token available at `HTTP01ChallengePath(token)` by creating a file in the given s3 bucket. +func (s *HTTPProvider) Present(domain, token, keyAuth string) error { + ctx := context.Background() + + params := &s3.PutObjectInput{ + ACL: "public-read", + Bucket: aws.String(s.bucket), + Key: aws.String(strings.Trim(http01.ChallengePath(token), "/")), + Body: bytes.NewReader([]byte(keyAuth)), + } + + _, err := s.client.PutObject(ctx, params) + if err != nil { + return fmt.Errorf("s3: failed to upload token to s3: %w", err) + } + return nil +} + +// CleanUp removes the file created for the challenge. +func (s *HTTPProvider) CleanUp(domain, token, keyAuth string) error { + ctx := context.Background() + + params := &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(strings.Trim(http01.ChallengePath(token), "/")), + } + + _, err := s.client.DeleteObject(ctx, params) + if err != nil { + return fmt.Errorf("s3: could not remove file in s3 bucket after HTTP challenge: %w", err) + } + + return nil +} diff --git a/providers/http/s3/s3.toml b/providers/http/s3/s3.toml new file mode 100644 index 00000000..6e810766 --- /dev/null +++ b/providers/http/s3/s3.toml @@ -0,0 +1,54 @@ +Name = "Amazon S3" +Description = '''''' +URL = "https://aws.amazon.com/s3/" +Code = "s3" +Since = "v4.14.0" + +Example = ''' +AWS_ACCESS_KEY_ID=your_key_id \ +AWS_SECRET_ACCESS_KEY=your_secret_access_key \ +AWS_REGION=aws-region \ +lego --domains example.com --email your_example@email.com --http --http.s3-bucket your_s3_bucket --accept-tos=true run +''' + +Additional = ''' +## Description + +AWS Credentials are automatically detected in the following locations and prioritized in the following order: + +1. Environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, [`AWS_SESSION_TOKEN`] +2. Shared credentials file (defaults to `~/.aws/credentials`, profiles can be specified using `AWS_PROFILE`) +3. Amazon EC2 IAM role + +The AWS Region is automatically detected in the following locations and prioritized in the following order: + +1. Environment variables: `AWS_REGION` +2. Shared configuration file if `AWS_SDK_LOAD_CONFIG` is set (defaults to `~/.aws/config`, profiles can be specified using `AWS_PROFILE`) + +See also: https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/ + +### Broad privileges for testing purposes + +Will need to create an S3 bucket which has read permissions set for Everyone (public access). +The S3 bucket doesn't require static website hosting to be enabled. +AWS_REGION must match the region where the s3 bucket is hosted. +''' + +[Configuration] + [Configuration.Credentials] + AWS_ACCESS_KEY_ID = "Managed by the AWS client. Access key ID (`AWS_ACCESS_KEY_ID_FILE` is not supported, use `AWS_SHARED_CREDENTIALS_FILE` instead)" + AWS_SECRET_ACCESS_KEY = "Managed by the AWS client. Secret access key (`AWS_SECRET_ACCESS_KEY_FILE` is not supported, use `AWS_SHARED_CREDENTIALS_FILE` instead)" + AWS_REGION = "Managed by the AWS client (`AWS_REGION_FILE` is not supported)" + S3_BUCKET = "Name of the s3 bucket" + AWS_PROFILE = "Managed by the AWS client (`AWS_PROFILE_FILE` is not supported)" + AWS_SDK_LOAD_CONFIG = "Managed by the AWS client. Retrieve the region from the CLI config file (`AWS_SDK_LOAD_CONFIG_FILE` is not supported)" + AWS_ASSUME_ROLE_ARN = "Managed by the AWS Role ARN (`AWS_ASSUME_ROLE_ARN_FILE` is not supported)" + AWS_EXTERNAL_ID = "Managed by STS AssumeRole API operation (`AWS_EXTERNAL_ID_FILE` is not supported)" + [Configuration.Additional] + AWS_SHARED_CREDENTIALS_FILE = "Managed by the AWS client. Shared credentials file." + AWS_MAX_RETRIES = "The number of maximum returns the service will use to make an individual API request" + +[Links] + API = "https://docs.aws.amazon.com/AmazonS3/latest/userguide//Welcome.html" + GoClient = "https://docs.aws.amazon.com/sdk-for-go/" + diff --git a/providers/http/s3/s3_test.go b/providers/http/s3/s3_test.go new file mode 100644 index 00000000..1511fc2d --- /dev/null +++ b/providers/http/s3/s3_test.go @@ -0,0 +1,80 @@ +// Package s3 implements a HTTP provider for solving the HTTP-01 challenge +// using AWS S3 in combination with AWS CloudFront. +package s3 + +import ( + "fmt" + "io" + "net/http" + "os" + "testing" + + "github.com/go-acme/lego/v4/challenge/http01" + "github.com/go-acme/lego/v4/platform/tester" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + domain = "example.com" + token = "foo" + keyAuth = "bar" +) + +var envTest = tester.NewEnvTest( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION", + "S3_BUCKET") + +func TestLiveNewHTTPProvider_Valid(t *testing.T) { + if !envTest.IsLiveTest() { + t.Skip("skipping live test") + } + + envTest.RestoreEnv() + + _, err := NewHTTPProvider(envTest.GetValue("S3_BUCKET")) + require.NoError(t, err) +} + +func TestLiveNewHTTPProvider(t *testing.T) { + if !envTest.IsLiveTest() { + t.Skip("skipping live test") + } + + envTest.RestoreEnv() + + s3Bucket := os.Getenv("S3_BUCKET") + + provider, err := NewHTTPProvider(s3Bucket) + require.NoError(t, err) + + // Present + + err = provider.Present(domain, token, keyAuth) + require.NoError(t, err) + + chlgPath := fmt.Sprintf("http://%s.s3.%s.amazonaws.com%s", + s3Bucket, envTest.GetValue("AWS_REGION"), http01.ChallengePath(token)) + + resp, err := http.Get(chlgPath) + require.NoError(t, err) + + defer func() { _ = resp.Body.Close() }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + assert.Equal(t, []byte(keyAuth), data) + + // CleanUp + + err = provider.CleanUp(domain, token, keyAuth) + require.NoError(t, err) + + cleanupResp, err := http.Get(chlgPath) + require.NoError(t, err) + + assert.Equal(t, cleanupResp.StatusCode, 403) +}