Merge pull request #174 from janeczku/route53-aws-sdk

Switch route53 provider to the official AWS SDK

README.md

@@ -144,14 +144,19 @@ Replace `<INSERT_YOUR_HOSTED_ZONE_ID_HERE>` with the Route 53 zone ID of the dom
     "Statement": [
         {
             "Effect": "Allow",
-            "Action": [ "route53:ListHostedZones", "route53:GetChange" ],
+            "Action": [
+                "route53:GetChange",
+                "route53:ListHostedZonesByName"
+            ],
             "Resource": [
                 "*"
             ]
         },
         {
             "Effect": "Allow",
-            "Action": ["route53:ChangeResourceRecordSets"],
+            "Action": [
+                "route53:ChangeResourceRecordSets"
+            ],
             "Resource": [
                 "arn:aws:route53:::hostedzone/<INSERT_YOUR_HOSTED_ZONE_ID_HERE>"
             ]

providers/dns/route53/fixtures_test.go

@@ -0,0 +1,39 @@
+package route53
+
+var ChangeResourceRecordSetsResponse = `<?xml version="1.0" encoding="UTF-8"?>
+<ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
+<ChangeInfo>
+   <Id>/change/123456</Id>
+   <Status>PENDING</Status>
+   <SubmittedAt>2016-02-10T01:36:41.958Z</SubmittedAt>
+</ChangeInfo>
+</ChangeResourceRecordSetsResponse>`
+
+var ListHostedZonesByNameResponse = `<?xml version="1.0" encoding="UTF-8"?>
+<ListHostedZonesByNameResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
+   <HostedZones>
+      <HostedZone>
+         <Id>/hostedzone/ABCDEFG</Id>
+         <Name>example.com.</Name>
+         <CallerReference>D2224C5B-684A-DB4A-BB9A-E09E3BAFEA7A</CallerReference>
+         <Config>
+            <Comment>Test comment</Comment>
+            <PrivateZone>false</PrivateZone>
+         </Config>
+         <ResourceRecordSetCount>10</ResourceRecordSetCount>
+      </HostedZone>
+   </HostedZones>
+   <IsTruncated>true</IsTruncated>
+   <NextDNSName>example2.com</NextDNSName>
+   <NextHostedZoneId>ZLT12321321124</NextHostedZoneId>
+   <MaxItems>1</MaxItems>
+</ListHostedZonesByNameResponse>`
+
+var GetChangeResponse = `<?xml version="1.0" encoding="UTF-8"?>
+<GetChangeResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
+   <ChangeInfo>
+      <Id>123456</Id>
+      <Status>INSYNC</Status>
+      <SubmittedAt>2016-02-10T01:36:41.958Z</SubmittedAt>
+   </ChangeInfo>
+</GetChangeResponse>`

providers/dns/route53/route53.go

@@ -1,57 +1,69 @@
 // Package route53 implements a DNS provider for solving the DNS-01 challenge
-// using route53 DNS.
+// using AWS Route 53 DNS.
 package route53
 
 import (
 	"fmt"
-	"os"
+	"math/rand"
 	"strings"
 	"time"
 
-	"github.com/mitchellh/goamz/aws"
+	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mitchellh/goamz/route53"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/xenolf/lego/acme"
 )
 
-// DNSProvider is an implementation of the acme.ChallengeProvider interface
+const (
+	maxRetries = 5
+)
+
+// DNSProvider implements the acme.ChallengeProvider interface
 type DNSProvider struct {
 	client *route53.Route53
 }
 
+// customRetryer implements the client.Retryer interface by composing the
+// DefaultRetryer. It controls the logic for retrying recoverable request
+// errors (e.g. when rate limits are exceeded).
+type customRetryer struct {
+	client.DefaultRetryer
+}
+
+// RetryRules overwrites the DefaultRetryer's method.
+// It uses a basic exponential backoff algorithm that returns an initial
+// delay of ~400ms with an upper limit of ~30 seconds which should prevent
+// causing a high number of consecutive throttling errors.
+// For reference: Route 53 enforces an account-wide(!) 5req/s query limit.
+func (d customRetryer) RetryRules(r *request.Request) time.Duration {
+	retryCount := r.RetryCount
+	if retryCount > 7 {
+		retryCount = 7
+	}
+
+	delay := (1 << uint(retryCount)) * (rand.Intn(50) + 200)
+	return time.Duration(delay) * time.Millisecond
+}
+
 // NewDNSProvider returns a DNSProvider instance configured for the AWS
-// route53 service. The AWS region name must be passed in the environment
+// Route 53 service.
-// variable AWS_REGION.
+//
+// AWS Credentials are automatically detected in the following locations
+// and prioritized in the following order:
+// 1. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
+//    AWS_REGION, [AWS_SESSION_TOKEN]
+// 2. Shared credentials file (defaults to ~/.aws/credentials)
+// 3. Amazon EC2 IAM role
+//
+// See also: https://github.com/aws/aws-sdk-go/wiki/configuring-sdk
 func NewDNSProvider() (*DNSProvider, error) {
-	regionName := os.Getenv("AWS_REGION")
+	r := customRetryer{}
-	return NewDNSProviderCredentials("", "", regionName)
+	r.NumMaxRetries = maxRetries
-}
+	config := request.WithRetryer(aws.NewConfig(), r)
+	client := route53.New(session.New(config))
 
-// NewDNSProviderCredentials uses the supplied credentials to return a
-// DNSProvider instance configured for the AWS route53 service. Authentication
-// is done using the passed credentials or, if empty, falling back to the
-// custonmary AWS credential mechanisms, including the file referenced by
-// $AWS_CREDENTIAL_FILE (defaulting to $HOME/.aws/credentials) optionally
-// scoped to $AWS_PROFILE, credentials supplied by the environment variables
-// AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY [ + AWS_SECURITY_TOKEN ], and
-// finally credentials available via the EC2 instance metadata service.
-func NewDNSProviderCredentials(accessKey, secretKey, regionName string) (*DNSProvider, error) {
-	region, ok := aws.Regions[regionName]
-	if !ok {
-		return nil, fmt.Errorf("Invalid AWS region name %s", regionName)
-	}
-
-	// use aws.GetAuth, which tries really hard to find credentails:
-	//   - uses accessKey and secretKey, if provided
-	//   - uses AWS_PROFILE / AWS_CREDENTIAL_FILE, if provided
-	//   - uses AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY and optionally AWS_SECURITY_TOKEN, if provided
-	//   - uses EC2 instance metadata credentials (http://169.254.169.254/latest/meta-data/…), if available
-	//  ...and otherwise returns an error
-	auth, err := aws.GetAuth(accessKey, secretKey)
-	if err != nil {
-		return nil, err
-	}
-
-	client := route53.New(auth, region)
 	return &DNSProvider{client: client}, nil
 }
 
@@ -74,21 +86,37 @@ func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error {
 	if err != nil {
 		return err
 	}
+
 	recordSet := newTXTRecordSet(fqdn, value, ttl)
-	update := route53.Change{Action: action, Record: recordSet}
+	reqParams := &route53.ChangeResourceRecordSetsInput{
-	changes := []route53.Change{update}
+		HostedZoneId: aws.String(hostedZoneID),
-	req := route53.ChangeResourceRecordSetsRequest{Comment: "Created by Lego", Changes: changes}
+		ChangeBatch: &route53.ChangeBatch{
-	resp, err := r.client.ChangeResourceRecordSets(hostedZoneID, &req)
+			Comment: aws.String("Managed by Lego"),
+			Changes: []*route53.Change{
+				{
+					Action:            aws.String(action),
+					ResourceRecordSet: recordSet,
+				},
+			},
+		},
+	}
+
+	resp, err := r.client.ChangeResourceRecordSets(reqParams)
 	if err != nil {
 		return err
 	}
 
-	return acme.WaitFor(90*time.Second, 5*time.Second, func() (bool, error) {
+	statusId := resp.ChangeInfo.Id
-		status, err := r.client.GetChange(resp.ChangeInfo.ID)
+
+	return acme.WaitFor(120*time.Second, 4*time.Second, func() (bool, error) {
+		reqParams := &route53.GetChangeInput{
+			Id: statusId,
+		}
+		resp, err := r.client.GetChange(reqParams)
 		if err != nil {
 			return false, err
 		}
-		if status == "INSYNC" {
+		if *resp.ChangeInfo.Status == route53.ChangeStatusInsync {
 			return true, nil
 		}
 		return false, nil
@@ -96,59 +124,41 @@ func (r *DNSProvider) changeRecord(action, fqdn, value string, ttl int) error {
 }
 
 func (r *DNSProvider) getHostedZoneID(fqdn string) (string, error) {
-	zones := []route53.HostedZone{}
-	zoneResp, err := r.client.ListHostedZones("", 0)
-	if err != nil {
-		return "", err
-	}
-	zones = append(zones, zoneResp.HostedZones...)
-
-	for zoneResp.IsTruncated {
-		resp, err := r.client.ListHostedZones(zoneResp.Marker, 0)
-		if err != nil {
-			if rateExceeded(err) {
-				time.Sleep(time.Second)
-				continue
-			}
-			return "", err
-		}
-		zoneResp = resp
-		zones = append(zones, zoneResp.HostedZones...)
-	}
-
 	authZone, err := acme.FindZoneByFqdn(fqdn, acme.RecursiveNameserver)
 	if err != nil {
 		return "", err
 	}
 
-	var hostedZone route53.HostedZone
+	// .DNSName should not have a trailing dot
-	for _, zone := range zones {
+	reqParams := &route53.ListHostedZonesByNameInput{
-		if zone.Name == authZone {
+		DNSName:  aws.String(acme.UnFqdn(authZone)),
-			hostedZone = zone
+		MaxItems: aws.String("1"),
 	}
+	resp, err := r.client.ListHostedZonesByName(reqParams)
+	if err != nil {
+		return "", err
 	}
-	if hostedZone.ID == "" {
+
+	// .Name has a trailing dot
+	if len(resp.HostedZones) == 0 || *resp.HostedZones[0].Name != authZone {
 		return "", fmt.Errorf("Zone %s not found in Route53 for domain %s", authZone, fqdn)
 	}
 
-	return hostedZone.ID, nil
+	zoneId := *resp.HostedZones[0].Id
+	if strings.HasPrefix(zoneId, "/hostedzone/") {
+		zoneId = strings.TrimPrefix(zoneId, "/hostedzone/")
 	}
 
-func newTXTRecordSet(fqdn, value string, ttl int) route53.ResourceRecordSet {
+	return zoneId, nil
-	return route53.ResourceRecordSet{
-		Name:    fqdn,
-		Type:    "TXT",
-		Records: []string{value},
-		TTL:     ttl,
 }
 
+func newTXTRecordSet(fqdn, value string, ttl int) *route53.ResourceRecordSet {
+	return &route53.ResourceRecordSet{
+		Name: aws.String(fqdn),
+		Type: aws.String("TXT"),
+		TTL:  aws.Int64(int64(ttl)),
+		ResourceRecords: []*route53.ResourceRecord{
+			{Value: aws.String(value)},
+		},
 	}
-
-// Route53 API has pretty strict rate limits (5req/s globally per account)
-// Hence we check if we are being throttled to maybe retry the request
-func rateExceeded(err error) bool {
-	if strings.Contains(err.Error(), "Throttling") {
-		return true
-	}
-	return false
 }

providers/dns/route53/route53_test.go

@@ -1,160 +1,87 @@
 package route53
 
 import (
-	"net/http"
+	"net/http/httptest"
 	"os"
 	"testing"
-	"time"
 
-	"github.com/mitchellh/goamz/aws"
+	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mitchellh/goamz/route53"
+	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/mitchellh/goamz/testutil"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/stretchr/testify/assert"
 )
 
 var (
 	route53Secret string
 	route53Key    string
-	awsCredentialFile string
+	route53Region string
-	homeDir           string
-	testServer        *testutil.HTTPServer
 )
 
-var ChangeResourceRecordSetsAnswer = `<?xml version="1.0" encoding="UTF-8"?>
-<ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
-   <ChangeInfo>
-      <Id>/change/asdf</Id>
-      <Status>PENDING</Status>
-      <SubmittedAt>2014</SubmittedAt>
-   </ChangeInfo>
-</ChangeResourceRecordSetsResponse>`
-
-var ListHostedZonesAnswer = `<?xml version="1.0" encoding="utf-8"?>
-<ListHostedZonesResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
-    <HostedZones>
-        <HostedZone>
-            <Id>/hostedzone/Z2K123214213123</Id>
-            <Name>example.com.</Name>
-            <CallerReference>D2224C5B-684A-DB4A-BB9A-E09E3BAFEA7A</CallerReference>
-            <Config>
-                <Comment>Test comment</Comment>
-            </Config>
-            <ResourceRecordSetCount>10</ResourceRecordSetCount>
-        </HostedZone>
-        <HostedZone>
-            <Id>/hostedzone/ZLT12321321124</Id>
-            <Name>sub.example.com.</Name>
-            <CallerReference>A970F076-FCB1-D959-B395-96474CC84EB8</CallerReference>
-            <Config>
-                <Comment>Test comment for subdomain host</Comment>
-            </Config>
-            <ResourceRecordSetCount>4</ResourceRecordSetCount>
-        </HostedZone>
-    </HostedZones>
-    <IsTruncated>false</IsTruncated>
-    <MaxItems>100</MaxItems>
-</ListHostedZonesResponse>`
-
-var GetChangeAnswer = `<?xml version="1.0" encoding="UTF-8"?>
-<GetChangeResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
-   <ChangeInfo>
-      <Id>/change/asdf</Id>
-      <Status>INSYNC</Status>
-      <SubmittedAt>2016-02-03T01:36:41.958Z</SubmittedAt>
-   </ChangeInfo>
-</GetChangeResponse>`
-
-var serverResponseMap = testutil.ResponseMap{
-	"/2013-04-01/hostedzone/":                      testutil.Response{Status: 200, Headers: nil, Body: ListHostedZonesAnswer},
-	"/2013-04-01/hostedzone/Z2K123214213123/rrset": testutil.Response{Status: 200, Headers: nil, Body: ChangeResourceRecordSetsAnswer},
-	"/2013-04-01/change/asdf":                      testutil.Response{Status: 200, Headers: nil, Body: GetChangeAnswer},
-}
-
 func init() {
 	route53Key = os.Getenv("AWS_ACCESS_KEY_ID")
 	route53Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
-	awsCredentialFile = os.Getenv("AWS_CREDENTIAL_FILE")
+	route53Region = os.Getenv("AWS_REGION")
-	homeDir = os.Getenv("HOME")
-	testServer = testutil.NewHTTPServer()
-	testServer.Start()
 }
 
 func restoreRoute53Env() {
 	os.Setenv("AWS_ACCESS_KEY_ID", route53Key)
 	os.Setenv("AWS_SECRET_ACCESS_KEY", route53Secret)
-	os.Setenv("AWS_CREDENTIAL_FILE", awsCredentialFile)
+	os.Setenv("AWS_REGION", route53Region)
-	os.Setenv("HOME", homeDir)
 }
 
-func makeRoute53TestServer() *testutil.HTTPServer {
+func makeRoute53Provider(ts *httptest.Server) *DNSProvider {
-	testServer.Flush()
+	config := &aws.Config{
-	return testServer
+		Credentials: credentials.NewStaticCredentials("abc", "123", " "),
+		Endpoint:    aws.String(ts.URL),
+		Region:      aws.String("mock-region"),
+		MaxRetries:  aws.Int(1),
 	}
 
-func makeRoute53Provider(server *testutil.HTTPServer) *DNSProvider {
+	client := route53.New(session.New(config))
-	auth := aws.Auth{AccessKey: "abc", SecretKey: "123", Token: ""}
-	client := route53.NewWithClient(auth, aws.Region{Route53Endpoint: server.URL}, testutil.DefaultClient)
 	return &DNSProvider{client: client}
 }
 
-func TestNewDNSProviderValid(t *testing.T) {
+func TestCredentialsFromEnv(t *testing.T) {
-	os.Setenv("AWS_ACCESS_KEY_ID", "")
-	os.Setenv("AWS_SECRET_ACCESS_KEY", "")
-	os.Setenv("AWS_REGION", "")
-	_, err := NewDNSProviderCredentials("123", "123", "us-east-1")
-	assert.NoError(t, err)
-	restoreRoute53Env()
-}
-
-func TestNewDNSProviderValidEnv(t *testing.T) {
 	os.Setenv("AWS_ACCESS_KEY_ID", "123")
 	os.Setenv("AWS_SECRET_ACCESS_KEY", "123")
 	os.Setenv("AWS_REGION", "us-east-1")
-	_, err := NewDNSProvider()
+
-	assert.NoError(t, err)
+	config := &aws.Config{
+		CredentialsChainVerboseErrors: aws.Bool(true),
+	}
+
+	sess := session.New(config)
+	_, err := sess.Config.Credentials.Get()
+	assert.NoError(t, err, "Expected credentials to be set from environment")
+
 	restoreRoute53Env()
 }
 
-func TestNewDNSProviderMissingAuthErr(t *testing.T) {
+func TestRegionFromEnv(t *testing.T) {
-	os.Setenv("AWS_ACCESS_KEY_ID", "")
+	os.Setenv("AWS_REGION", "us-east-1")
-	os.Setenv("AWS_SECRET_ACCESS_KEY", "")
-	os.Setenv("AWS_CREDENTIAL_FILE", "") // in case test machine has this variable set
-	os.Setenv("HOME", "/")               // in case test machine has ~/.aws/credentials
 
-	// The default AWS HTTP client retries three times with a deadline of 10 seconds.
+	sess := session.New(aws.NewConfig())
-	// Replace the default HTTP client with one that does not retry and has a low timeout.
+	assert.Equal(t, "us-east-1", *sess.Config.Region, "Expected Region to be set from environment")
-	awsClient := aws.RetryingClient
-	aws.RetryingClient = &http.Client{Timeout: time.Millisecond}
 
-	_, err := NewDNSProviderCredentials("", "", "us-east-1")
-	assert.EqualError(t, err, "No valid AWS authentication found")
 	restoreRoute53Env()
-
-	// restore default AWS HTTP client
-	aws.RetryingClient = awsClient
-}
-
-func TestNewDNSProviderInvalidRegionErr(t *testing.T) {
-	_, err := NewDNSProviderCredentials("123", "123", "us-east-3")
-	assert.EqualError(t, err, "Invalid AWS region name us-east-3")
 }
 
 func TestRoute53Present(t *testing.T) {
-	assert := assert.New(t)
+	mockResponses := MockResponseMap{
-	testServer := makeRoute53TestServer()
+		"/2013-04-01/hostedzonesbyname":         MockResponse{StatusCode: 200, Body: ListHostedZonesByNameResponse},
-	provider := makeRoute53Provider(testServer)
+		"/2013-04-01/hostedzone/ABCDEFG/rrset/": MockResponse{StatusCode: 200, Body: ChangeResourceRecordSetsResponse},
-	testServer.ResponseMap(3, serverResponseMap)
+		"/2013-04-01/change/123456":             MockResponse{StatusCode: 200, Body: GetChangeResponse},
+	}
+
+	ts := newMockServer(t, mockResponses)
+	defer ts.Close()
+
+	provider := makeRoute53Provider(ts)
 
 	domain := "example.com"
 	keyAuth := "123456d=="
 
 	err := provider.Present(domain, "", keyAuth)
-	assert.NoError(err, "Expected Present to return no error")
+	assert.NoError(t, err, "Expected Present to return no error")
-
-	httpReqs := testServer.WaitRequests(3)
-	httpReq := httpReqs[1]
-
-	assert.Equal("/2013-04-01/hostedzone/Z2K123214213123/rrset", httpReq.URL.Path,
-		"Expected Present to select the correct hostedzone")
-
 }

providers/dns/route53/testutil_test.go

@@ -0,0 +1,38 @@
+package route53
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// MockResponse represents a predefined response used by a mock server
+type MockResponse struct {
+	StatusCode int
+	Body       string
+}
+
+// MockResponseMap maps request paths to responses
+type MockResponseMap map[string]MockResponse
+
+func newMockServer(t *testing.T, responses MockResponseMap) *httptest.Server {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path := r.URL.Path
+		resp, ok := responses[path]
+		if !ok {
+			msg := fmt.Sprintf("Requested path not found in response map: %s", path)
+			require.FailNow(t, msg)
+		}
+
+		w.Header().Set("Content-Type", "application/xml")
+		w.WriteHeader(resp.StatusCode)
+		w.Write([]byte(resp.Body))
+	}))
+
+	time.Sleep(100 * time.Millisecond)
+	return ts
+}