From 9f1b9e39af0b2e3552d2be99a0333dbe21b3dae7 Mon Sep 17 00:00:00 2001 From: Jan Broer Date: Sat, 26 Mar 2016 04:34:31 +0100 Subject: [PATCH] Switch route53 provider to the official AWS SDK Fully backwards compatible in terms of credential mechanisms (environment variables, shared credentials file, EC2 metadata). If a custom AWS IAM policy is in use it needs to be updated with permissions for the route53:ListHostedZonesByName action. --- README.md | 9 +- providers/dns/route53/fixtures_test.go | 39 ++++++ providers/dns/route53/route53.go | 174 +++++++++++++------------ providers/dns/route53/route53_test.go | 159 ++++++---------------- providers/dns/route53/testutil_test.go | 38 ++++++ 5 files changed, 219 insertions(+), 200 deletions(-) create mode 100644 providers/dns/route53/fixtures_test.go create mode 100644 providers/dns/route53/testutil_test.go diff --git a/README.md b/README.md index b59e5e18..b643218b 100644 --- a/README.md +++ b/README.md @@ -143,14 +143,19 @@ Replace `` with the Route 53 zone ID of the dom "Statement": [ { "Effect": "Allow", - "Action": [ "route53:ListHostedZones", "route53:GetChange" ], + "Action": [ + "route53:GetChange", + "route53:ListHostedZonesByName" + ], "Resource": [ "*" ] }, { "Effect": "Allow", - "Action": ["route53:ChangeResourceRecordSets"], + "Action": [ + "route53:ChangeResourceRecordSets" + ], "Resource": [ "arn:aws:route53:::hostedzone/" ] diff --git a/providers/dns/route53/fixtures_test.go b/providers/dns/route53/fixtures_test.go new file mode 100644 index 00000000..a5cc9c87 --- /dev/null +++ b/providers/dns/route53/fixtures_test.go @@ -0,0 +1,39 @@ +package route53 + +var ChangeResourceRecordSetsResponse = ` + + + /change/123456 + PENDING + 2016-02-10T01:36:41.958Z + +` + +var ListHostedZonesByNameResponse = ` + + + + /hostedzone/ABCDEFG + example.com. + D2224C5B-684A-DB4A-BB9A-E09E3BAFEA7A + + Test comment + false + + 10 + + + true + example2.com + ZLT12321321124 + 1 +` + +var GetChangeResponse = ` + + + 123456 + INSYNC + 2016-02-10T01:36:41.958Z + +` diff --git a/providers/dns/route53/route53.go b/providers/dns/route53/route53.go index 57fc53c0..47b56176 100644 --- a/providers/dns/route53/route53.go +++ b/providers/dns/route53/route53.go @@ -1,57 +1,69 @@ // Package route53 implements a DNS provider for solving the DNS-01 challenge -// using route53 DNS. +// using AWS Route 53 DNS. package route53 import ( "fmt" - "os" + "math/rand" "strings" "time" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/route53" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/route53" "github.com/xenolf/lego/acme" ) -// DNSProvider is an implementation of the acme.ChallengeProvider interface +const ( + maxRetries = 5 +) + +// DNSProvider implements the acme.ChallengeProvider interface type DNSProvider struct { client *route53.Route53 } -// NewDNSProvider returns a DNSProvider instance configured for the AWS -// route53 service. The AWS region name must be passed in the environment -// variable AWS_REGION. -func NewDNSProvider() (*DNSProvider, error) { - regionName := os.Getenv("AWS_REGION") - return NewDNSProviderCredentials("", "", regionName) +// customRetryer implements the client.Retryer interface by composing the +// DefaultRetryer. It controls the logic for retrying recoverable request +// errors (e.g. when rate limits are exceeded). +type customRetryer struct { + client.DefaultRetryer } -// NewDNSProviderCredentials uses the supplied credentials to return a -// DNSProvider instance configured for the AWS route53 service. Authentication -// is done using the passed credentials or, if empty, falling back to the -// custonmary AWS credential mechanisms, including the file referenced by -// $AWS_CREDENTIAL_FILE (defaulting to $HOME/.aws/credentials) optionally -// scoped to $AWS_PROFILE, credentials supplied by the environment variables -// AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY [ + AWS_SECURITY_TOKEN ], and -// finally credentials available via the EC2 instance metadata service. -func NewDNSProviderCredentials(accessKey, secretKey, regionName string) (*DNSProvider, error) { - region, ok := aws.Regions[regionName] - if !ok { - return nil, fmt.Errorf("Invalid AWS region name %s", regionName) +// RetryRules overwrites the DefaultRetryer's method. +// It uses a basic exponential backoff algorithm that returns an initial +// delay of ~400ms with an upper limit of ~30 seconds which should prevent +// causing a high number of consecutive throttling errors. +// For reference: Route 53 enforces an account-wide(!) 5req/s query limit. +func (d customRetryer) RetryRules(r *request.Request) time.Duration { + retryCount := r.RetryCount + if retryCount > 7 { + retryCount = 7 } - // use aws.GetAuth, which tries really hard to find credentails: - // - uses accessKey and secretKey, if provided - // - uses AWS_PROFILE / AWS_CREDENTIAL_FILE, if provided - // - uses AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY and optionally AWS_SECURITY_TOKEN, if provided - // - uses EC2 instance metadata credentials (http://169.254.169.254/latest/meta-data/…), if available - // ...and otherwise returns an error - auth, err := aws.GetAuth(accessKey, secretKey) - if err != nil { - return nil, err - } + delay := (1 << uint(retryCount)) * (rand.Intn(50) + 200) + return time.Duration(delay) * time.Millisecond +} + +// NewDNSProvider returns a DNSProvider instance configured for the AWS +// Route 53 service. +// +// AWS Credentials are automatically detected in the following locations +// and prioritized in the following order: +// 1. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +// AWS_REGION, [AWS_SESSION_TOKEN] +// 2. Shared credentials file (defaults to ~/.aws/credentials) +// 3. Amazon EC2 IAM role +// +// See also: https://github.com/aws/aws-sdk-go/wiki/configuring-sdk +func NewDNSProvider() (*DNSProvider, error) { + r := customRetryer{} + r.NumMaxRetries = maxRetries + config := request.WithRetryer(aws.NewConfig(), r) + client := route53.New(session.New(config)) - client := route53.New(auth, region) return &DNSProvider{client: client}, nil } @@ -74,21 +86,37 @@ func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error { if err != nil { return err } + recordSet := newTXTRecordSet(fqdn, value, ttl) - update := route53.Change{Action: action, Record: recordSet} - changes := []route53.Change{update} - req := route53.ChangeResourceRecordSetsRequest{Comment: "Created by Lego", Changes: changes} - resp, err := r.client.ChangeResourceRecordSets(hostedZoneID, &req) + reqParams := &route53.ChangeResourceRecordSetsInput{ + HostedZoneId: aws.String(hostedZoneID), + ChangeBatch: &route53.ChangeBatch{ + Comment: aws.String("Managed by Lego"), + Changes: []*route53.Change{ + { + Action: aws.String(action), + ResourceRecordSet: recordSet, + }, + }, + }, + } + + resp, err := r.client.ChangeResourceRecordSets(reqParams) if err != nil { return err } - return acme.WaitFor(90*time.Second, 5*time.Second, func() (bool, error) { - status, err := r.client.GetChange(resp.ChangeInfo.ID) + statusId := resp.ChangeInfo.Id + + return acme.WaitFor(120*time.Second, 4*time.Second, func() (bool, error) { + reqParams := &route53.GetChangeInput{ + Id: statusId, + } + resp, err := r.client.GetChange(reqParams) if err != nil { return false, err } - if status == "INSYNC" { + if *resp.ChangeInfo.Status == route53.ChangeStatusInsync { return true, nil } return false, nil @@ -96,59 +124,41 @@ func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error { } func (r *DNSProvider) getHostedZoneID(fqdn string) (string, error) { - zones := []route53.HostedZone{} - zoneResp, err := r.client.ListHostedZones("", 0) - if err != nil { - return "", err - } - zones = append(zones, zoneResp.HostedZones...) - - for zoneResp.IsTruncated { - resp, err := r.client.ListHostedZones(zoneResp.Marker, 0) - if err != nil { - if rateExceeded(err) { - time.Sleep(time.Second) - continue - } - return "", err - } - zoneResp = resp - zones = append(zones, zoneResp.HostedZones...) - } - authZone, err := acme.FindZoneByFqdn(fqdn, acme.RecursiveNameserver) if err != nil { return "", err } - var hostedZone route53.HostedZone - for _, zone := range zones { - if zone.Name == authZone { - hostedZone = zone - } + // .DNSName should not have a trailing dot + reqParams := &route53.ListHostedZonesByNameInput{ + DNSName: aws.String(acme.UnFqdn(authZone)), + MaxItems: aws.String("1"), } - if hostedZone.ID == "" { + resp, err := r.client.ListHostedZonesByName(reqParams) + if err != nil { + return "", err + } + + // .Name has a trailing dot + if len(resp.HostedZones) == 0 || *resp.HostedZones[0].Name != authZone { return "", fmt.Errorf("Zone %s not found in Route53 for domain %s", authZone, fqdn) } - return hostedZone.ID, nil -} - -func newTXTRecordSet(fqdn, value string, ttl int) route53.ResourceRecordSet { - return route53.ResourceRecordSet{ - Name: fqdn, - Type: "TXT", - Records: []string{value}, - TTL: ttl, + zoneId := *resp.HostedZones[0].Id + if strings.HasPrefix(zoneId, "/hostedzone/") { + zoneId = strings.TrimPrefix(zoneId, "/hostedzone/") } + return zoneId, nil } -// Route53 API has pretty strict rate limits (5req/s globally per account) -// Hence we check if we are being throttled to maybe retry the request -func rateExceeded(err error) bool { - if strings.Contains(err.Error(), "Throttling") { - return true +func newTXTRecordSet(fqdn, value string, ttl int) *route53.ResourceRecordSet { + return &route53.ResourceRecordSet{ + Name: aws.String(fqdn), + Type: aws.String("TXT"), + TTL: aws.Int64(int64(ttl)), + ResourceRecords: []*route53.ResourceRecord{ + {Value: aws.String(value)}, + }, } - return false } diff --git a/providers/dns/route53/route53_test.go b/providers/dns/route53/route53_test.go index 2b23f960..ab8739a5 100644 --- a/providers/dns/route53/route53_test.go +++ b/providers/dns/route53/route53_test.go @@ -1,160 +1,87 @@ package route53 import ( - "net/http" + "net/http/httptest" "os" "testing" - "time" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/route53" - "github.com/mitchellh/goamz/testutil" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/route53" "github.com/stretchr/testify/assert" ) var ( - route53Secret string - route53Key string - awsCredentialFile string - homeDir string - testServer *testutil.HTTPServer + route53Secret string + route53Key string + route53Region string ) -var ChangeResourceRecordSetsAnswer = ` - - - /change/asdf - PENDING - 2014 - -` - -var ListHostedZonesAnswer = ` - - - - /hostedzone/Z2K123214213123 - example.com. - D2224C5B-684A-DB4A-BB9A-E09E3BAFEA7A - - Test comment - - 10 - - - /hostedzone/ZLT12321321124 - sub.example.com. - A970F076-FCB1-D959-B395-96474CC84EB8 - - Test comment for subdomain host - - 4 - - - false - 100 -` - -var GetChangeAnswer = ` - - - /change/asdf - INSYNC - 2016-02-03T01:36:41.958Z - -` - -var serverResponseMap = testutil.ResponseMap{ - "/2013-04-01/hostedzone/": testutil.Response{Status: 200, Headers: nil, Body: ListHostedZonesAnswer}, - "/2013-04-01/hostedzone/Z2K123214213123/rrset": testutil.Response{Status: 200, Headers: nil, Body: ChangeResourceRecordSetsAnswer}, - "/2013-04-01/change/asdf": testutil.Response{Status: 200, Headers: nil, Body: GetChangeAnswer}, -} - func init() { route53Key = os.Getenv("AWS_ACCESS_KEY_ID") route53Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") - awsCredentialFile = os.Getenv("AWS_CREDENTIAL_FILE") - homeDir = os.Getenv("HOME") - testServer = testutil.NewHTTPServer() - testServer.Start() + route53Region = os.Getenv("AWS_REGION") } func restoreRoute53Env() { os.Setenv("AWS_ACCESS_KEY_ID", route53Key) os.Setenv("AWS_SECRET_ACCESS_KEY", route53Secret) - os.Setenv("AWS_CREDENTIAL_FILE", awsCredentialFile) - os.Setenv("HOME", homeDir) + os.Setenv("AWS_REGION", route53Region) } -func makeRoute53TestServer() *testutil.HTTPServer { - testServer.Flush() - return testServer -} +func makeRoute53Provider(ts *httptest.Server) *DNSProvider { + config := &aws.Config{ + Credentials: credentials.NewStaticCredentials("abc", "123", " "), + Endpoint: aws.String(ts.URL), + Region: aws.String("mock-region"), + MaxRetries: aws.Int(1), + } -func makeRoute53Provider(server *testutil.HTTPServer) *DNSProvider { - auth := aws.Auth{AccessKey: "abc", SecretKey: "123", Token: ""} - client := route53.NewWithClient(auth, aws.Region{Route53Endpoint: server.URL}, testutil.DefaultClient) + client := route53.New(session.New(config)) return &DNSProvider{client: client} } -func TestNewDNSProviderValid(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", "") - os.Setenv("AWS_SECRET_ACCESS_KEY", "") - os.Setenv("AWS_REGION", "") - _, err := NewDNSProviderCredentials("123", "123", "us-east-1") - assert.NoError(t, err) - restoreRoute53Env() -} - -func TestNewDNSProviderValidEnv(t *testing.T) { +func TestCredentialsFromEnv(t *testing.T) { os.Setenv("AWS_ACCESS_KEY_ID", "123") os.Setenv("AWS_SECRET_ACCESS_KEY", "123") os.Setenv("AWS_REGION", "us-east-1") - _, err := NewDNSProvider() - assert.NoError(t, err) + + config := &aws.Config{ + CredentialsChainVerboseErrors: aws.Bool(true), + } + + sess := session.New(config) + _, err := sess.Config.Credentials.Get() + assert.NoError(t, err, "Expected credentials to be set from environment") + restoreRoute53Env() } -func TestNewDNSProviderMissingAuthErr(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", "") - os.Setenv("AWS_SECRET_ACCESS_KEY", "") - os.Setenv("AWS_CREDENTIAL_FILE", "") // in case test machine has this variable set - os.Setenv("HOME", "/") // in case test machine has ~/.aws/credentials +func TestRegionFromEnv(t *testing.T) { + os.Setenv("AWS_REGION", "us-east-1") - // The default AWS HTTP client retries three times with a deadline of 10 seconds. - // Replace the default HTTP client with one that does not retry and has a low timeout. - awsClient := aws.RetryingClient - aws.RetryingClient = &http.Client{Timeout: time.Millisecond} + sess := session.New(aws.NewConfig()) + assert.Equal(t, "us-east-1", *sess.Config.Region, "Expected Region to be set from environment") - _, err := NewDNSProviderCredentials("", "", "us-east-1") - assert.EqualError(t, err, "No valid AWS authentication found") restoreRoute53Env() - - // restore default AWS HTTP client - aws.RetryingClient = awsClient -} - -func TestNewDNSProviderInvalidRegionErr(t *testing.T) { - _, err := NewDNSProviderCredentials("123", "123", "us-east-3") - assert.EqualError(t, err, "Invalid AWS region name us-east-3") } func TestRoute53Present(t *testing.T) { - assert := assert.New(t) - testServer := makeRoute53TestServer() - provider := makeRoute53Provider(testServer) - testServer.ResponseMap(3, serverResponseMap) + mockResponses := MockResponseMap{ + "/2013-04-01/hostedzonesbyname": MockResponse{StatusCode: 200, Body: ListHostedZonesByNameResponse}, + "/2013-04-01/hostedzone/ABCDEFG/rrset/": MockResponse{StatusCode: 200, Body: ChangeResourceRecordSetsResponse}, + "/2013-04-01/change/123456": MockResponse{StatusCode: 200, Body: GetChangeResponse}, + } + + ts := newMockServer(t, mockResponses) + defer ts.Close() + + provider := makeRoute53Provider(ts) domain := "example.com" keyAuth := "123456d==" err := provider.Present(domain, "", keyAuth) - assert.NoError(err, "Expected Present to return no error") - - httpReqs := testServer.WaitRequests(3) - httpReq := httpReqs[1] - - assert.Equal("/2013-04-01/hostedzone/Z2K123214213123/rrset", httpReq.URL.Path, - "Expected Present to select the correct hostedzone") - + assert.NoError(t, err, "Expected Present to return no error") } diff --git a/providers/dns/route53/testutil_test.go b/providers/dns/route53/testutil_test.go new file mode 100644 index 00000000..e448a685 --- /dev/null +++ b/providers/dns/route53/testutil_test.go @@ -0,0 +1,38 @@ +package route53 + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// MockResponse represents a predefined response used by a mock server +type MockResponse struct { + StatusCode int + Body string +} + +// MockResponseMap maps request paths to responses +type MockResponseMap map[string]MockResponse + +func newMockServer(t *testing.T, responses MockResponseMap) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + resp, ok := responses[path] + if !ok { + msg := fmt.Sprintf("Requested path not found in response map: %s", path) + require.FailNow(t, msg) + } + + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(resp.StatusCode) + w.Write([]byte(resp.Body)) + })) + + time.Sleep(100 * time.Millisecond) + return ts +}