Add automated challenge retries, RFC 8555

acme/api/handler.go

@@ -180,8 +180,9 @@ func (h *Handler) GetChallenge(w http.ResponseWriter, r *http.Request) {
 	ch, err = h.Auth.ValidateChallenge(prov, acc.GetID(), chID, acc.GetKey())
 	if err != nil {
 		api.WriteError(w, err)
-	} else if ch.Status != acme.StatusValid && ch.Status != acme.StatusInvalid {
+	} else if ch.Retry.Active {
-		w.Header().Add("Retry-After", "60")
+		retryAfter := int(ch.Retry.Backoffs) * (10 - ch.Retry.Called)
+		w.Header().Add("Retry-After", string(retryAfter))
 		api.JSON(w, ch)
 	} else {
 		getLink := h.Auth.GetLink

acme/api/handler_test.go

@@ -589,6 +589,7 @@ func ch() acme.Challenge {
 		URL:     "https://ca.smallstep.com/acme/challenge/chID",
 		ID:      "chID",
 		AuthzID: "authzID",
+		Retry:   &acme.Retry{Called:0, Backoffs:1, Active:false},
 	}
 }
 
@@ -733,6 +734,37 @@ func TestHandlerGetChallenge(t *testing.T) {
 				ch:         ch,
 			}
 		},
+		"ok/retry-after": func(t *testing.T) test {
+			key, err := jose.GenerateJWK("EC", "P-256", "ES256", "sig", "", 0)
+			assert.FatalError(t, err)
+			acc := &acme.Account{ID: "accID", Key: key}
+			ctx := context.WithValue(context.Background(), provisionerContextKey, prov)
+			ctx = context.WithValue(ctx, accContextKey, acc)
+			// TODO: Add correct key such that challenge object is already "active"
+			chiCtxInactive := chi.NewRouteContext()
+			chiCtxInactive.URLParams.Add("chID", "chID")
+			//chiCtxInactive.URLParams.Add("Active", "true")
+			ctx = context.WithValue(ctx, chi.RouteCtxKey, chiCtxInactive)
+			ch := ch()
+			ch.Retry.Active = true
+			chJSON, err := json.Marshal(ch)
+			assert.FatalError(t, err)
+			ctx = context.WithValue(ctx, payloadContextKey, &payloadInfo{value: chJSON})
+			return test{
+				auth: &mockAcmeAuthority{
+					validateChallenge: func(p provisioner.Interface, accID, id string, jwk *jose.JSONWebKey) (*acme.Challenge, error) {
+						assert.Equals(t, p, prov)
+						assert.Equals(t, accID, acc.ID)
+						assert.Equals(t, id, ch.ID)
+						assert.Equals(t, jwk.KeyID, key.KeyID)
+						return &ch, nil
+					},
+				},
+				ctx:        ctx,
+				statusCode: 200,
+				ch:         ch,
+			}
+		},
 	}
 	for name, run := range tests {
 		tc := run(t)
@@ -760,13 +792,17 @@ func TestHandlerGetChallenge(t *testing.T) {
 				assert.Equals(t, ae.Identifier, prob.Identifier)
 				assert.Equals(t, ae.Subproblems, prob.Subproblems)
 				assert.Equals(t, res.Header["Content-Type"], []string{"application/problem+json"})
-			} else {
+			} else if res.StatusCode >= 200 && assert.True(t,res.Header["Retry-After"] == nil){
 				expB, err := json.Marshal(tc.ch)
 				assert.FatalError(t, err)
 				assert.Equals(t, bytes.TrimSpace(body), expB)
 				assert.Equals(t, res.Header["Link"], []string{fmt.Sprintf("<https://ca.smallstep.com/acme/authz/%s>;rel=\"up\"", tc.ch.AuthzID)})
 				assert.Equals(t, res.Header["Location"], []string{url})
 				assert.Equals(t, res.Header["Content-Type"], []string{"application/json"})
+			} else {
+				expB, err := json.Marshal(tc.ch)
+				assert.FatalError(t, err)
+				assert.Equals(t, bytes.TrimSpace(body), expB)
 			}
 		})
 	}

acme/authority.go

@@ -5,6 +5,8 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
+	"math"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
@@ -263,21 +265,38 @@ func (a *Authority) ValidateChallenge(p provisioner.Interface, accID, chID strin
 	if accID != ch.getAccountID() {
 		return nil, UnauthorizedErr(errors.New("account does not own challenge"))
 	}
+	retry := ch.getRetry()
+	if retry.Active {
+		return ch.toACME(a.db, a.dir, p)
+	}
+	retry.Mux.Lock()
+	defer retry.Mux.Unlock()
+
 	client := http.Client{
 		Timeout: time.Duration(30 * time.Second),
 	}
 	dialer := &net.Dialer{
 		Timeout: 30 * time.Second,
 	}
-	ch, err = ch.validate(a.db, jwk, validateOptions{
+	for i:=0; i < 10; i++ {
-		httpGet:   client.Get,
+		ch, err = ch.validate(a.db, jwk, validateOptions{
-		lookupTxt: net.LookupTXT,
+			httpGet:   client.Get,
-		tlsDial: func(network, addr string, config *tls.Config) (*tls.Conn, error) {
+			lookupTxt: net.LookupTXT,
-			return tls.DialWithDialer(dialer, network, addr, config)
+			tlsDial: func(network, addr string, config *tls.Config) (*tls.Conn, error) {
-		},
+				return tls.DialWithDialer(dialer, network, addr, config)
-	})
+			},
-	if err != nil {
+		})
-		return nil, Wrap(err, "error attempting challenge validation")
+		if err != nil {
+			return nil, Wrap(err, "error attempting challenge validation")
+		}
+		if ch.getStatus() == StatusValid {
+			break
+		}
+		if ch.getStatus() == StatusInvalid {
+			return ch.toACME(a.db, a.dir, p)
+		}
+		duration := time.Duration(ch.getRetry().Backoffs + math.Mod(rand.Float64(), 5))
+		time.Sleep(duration*time.Second)
 	}
 	return ch.toACME(a.db, a.dir, p)
 }

acme/authority_test.go

@@ -1276,6 +1276,7 @@ func TestAuthorityValidateChallenge(t *testing.T) {
 			assert.Fatal(t, ok)
 			_ch.baseChallenge.Status = StatusValid
 			_ch.baseChallenge.Validated = clock.Now()
+			_ch.baseChallenge.Retry.Called = 0
 			b, err := json.Marshal(ch)
 			assert.FatalError(t, err)
 			auth, err := NewAuthority(&db.MockNoSQLDB{
@@ -1309,12 +1310,10 @@ func TestAuthorityValidateChallenge(t *testing.T) {
 				if assert.Nil(t, tc.err) {
 					gotb, err := json.Marshal(acmeCh)
 					assert.FatalError(t, err)
-
 					acmeExp, err := tc.ch.toACME(nil, tc.auth.dir, prov)
 					assert.FatalError(t, err)
 					expb, err := json.Marshal(acmeExp)
 					assert.FatalError(t, err)
-
 					assert.Equals(t, expb, gotb)
 				}
 			}

acme/challenge.go

@@ -14,6 +14,7 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -31,10 +32,18 @@ type Challenge struct {
 	Validated string  `json:"validated,omitempty"`
 	URL       string  `json:"url"`
 	Error     *AError `json:"error,omitempty"`
+	Retry     *Retry   `json:"retry"`
 	ID        string  `json:"-"`
 	AuthzID   string  `json:"-"`
 }
 
+type Retry struct {
+	Called    int         `json:"id"`
+	Backoffs  float64     `json:"backoffs"`
+	Active    bool        `json:"active"`
+	Mux	      sync.Mutex  `json:"mux"`
+}
+
 // ToLog enables response logging.
 func (c *Challenge) ToLog() (interface{}, error) {
 	b, err := json.Marshal(c)
@@ -75,6 +84,7 @@ type challenge interface {
 	getID() string
 	getAuthzID() string
 	getToken() string
+	getRetry() *Retry
 	clone() *baseChallenge
 	getAccountID() string
 	getValidated() time.Time
@@ -101,6 +111,7 @@ type baseChallenge struct {
 	Validated time.Time `json:"validated"`
 	Created   time.Time `json:"created"`
 	Error     *AError   `json:"error"`
+	Retry     *Retry     `json:"retry"`
 }
 
 func newBaseChallenge(accountID, authzID string) (*baseChallenge, error) {
@@ -120,6 +131,7 @@ func newBaseChallenge(accountID, authzID string) (*baseChallenge, error) {
 		Status:    StatusPending,
 		Token:     token,
 		Created:   clock.Now(),
+		Retry:     &Retry{Called:0, Backoffs:1, Active:false},
 	}, nil
 }
 
@@ -158,6 +170,11 @@ func (bc *baseChallenge) getToken() string {
 	return bc.Token
 }
 
+// getRetry returns the retry state of the baseChallenge
+func (bc *baseChallenge) getRetry() *Retry {
+	return bc.Retry
+}
+
 // getValidated returns the validated time of the baseChallenge.
 func (bc *baseChallenge) getValidated() time.Time {
 	return bc.Validated
@@ -190,6 +207,9 @@ func (bc *baseChallenge) toACME(db nosql.DB, dir *directory, p provisioner.Inter
 	if bc.Error != nil {
 		ac.Error = bc.Error
 	}
+	if bc.Retry != nil {
+		ac.Retry = bc.Retry
+	}
 	return ac, nil
 }
 
@@ -241,6 +261,11 @@ func (bc *baseChallenge) storeError(db nosql.DB, err *Error) error {
 	return clone.save(db, bc)
 }
 
+func (bc *baseChallenge) storeRetry(db nosql.DB, retry *Retry) error {
+	clone := bc.clone()
+	return clone.save(db, bc)
+}
+
 // unmarshalChallenge unmarshals a challenge type into the correct sub-type.
 func unmarshalChallenge(data []byte) (challenge, error) {
 	var getType struct {
@@ -303,7 +328,18 @@ func newHTTP01Challenge(db nosql.DB, ops ChallengeOptions) (challenge, error) {
 // updated.
 func (hc *http01Challenge) validate(db nosql.DB, jwk *jose.JSONWebKey, vo validateOptions) (challenge, error) {
 	// If already valid or invalid then return without performing validation.
-	if hc.getStatus() == StatusValid || hc.getStatus() == StatusInvalid {
+	if hc.getStatus() == StatusValid {
+		return hc, nil
+	}
+	if hc.getStatus() == StatusInvalid {
+		// TODO: Resolve segfault on upd.save
+		upd := hc.clone()
+		upd.Status = StatusPending
+		if err := upd.save(db, hc); err != nil {
+			fmt.Printf("Error in save: %s\n\n\n", err)
+			return nil, err
+		}
+		fmt.Print("Through Save\n\n")
 		return hc, nil
 	}
 	url := fmt.Sprintf("http://%s/.well-known/acme-challenge/%s", hc.Value, hc.Token)
@@ -343,10 +379,18 @@ func (hc *http01Challenge) validate(db nosql.DB, jwk *jose.JSONWebKey, vo valida
 		upd := hc.clone()
 		upd.Error = rejectedErr.ToACME()
 		upd.Error.Subproblems = append(upd.Error.Subproblems, rejectedErr)
+		upd.Retry.Called ++
+		upd.Retry.Active = true
+		if upd.Retry.Called >= 10 {
+			upd.Status = StatusInvalid
+			upd.Retry.Backoffs *= 2
+			upd.Retry.Active = false
+			upd.Retry.Called = 0
+		}
 		if err = upd.save(db, hc); err != nil {
 			return nil, err
 		}
-		return hc, nil
+		return hc, err
 	}
 
 	// Update and store the challenge.
@@ -354,6 +398,7 @@ func (hc *http01Challenge) validate(db nosql.DB, jwk *jose.JSONWebKey, vo valida
 	upd.Status = StatusValid
 	upd.Error = nil
 	upd.Validated = clock.Now()
+	upd.Retry.Active = false
 
 	if err := upd.save(db, hc); err != nil {
 		return nil, err
@@ -567,6 +612,14 @@ func (dc *dns01Challenge) validate(db nosql.DB, jwk *jose.JSONWebKey, vo validat
 		upd := dc.clone()
 		upd.Error = dnsErr.ToACME()
 		upd.Error.Subproblems = append(upd.Error.Subproblems, dnsErr)
+		upd.Retry.Called ++
+		upd.Retry.Active = true
+		if upd.Retry.Called >= 10 {
+			upd.Status = StatusInvalid
+			upd.Retry.Backoffs *= 2
+			upd.Retry.Active = false
+			upd.Retry.Called = 0
+		}
 		if err = upd.save(db, dc); err != nil {
 			return nil, err
 		}
@@ -602,6 +655,7 @@ func (dc *dns01Challenge) validate(db nosql.DB, jwk *jose.JSONWebKey, vo validat
 	upd.Status = StatusValid
 	upd.Error = nil
 	upd.Validated = time.Now().UTC()
+	upd.Retry.Active = false
 
 	if err := upd.save(db, dc); err != nil {
 		return nil, err

acme/challenge_test.go

@@ -772,7 +772,6 @@ func TestHTTP01Validate(t *testing.T) {
 			assert.FatalError(t, err)
 			oldb, err := json.Marshal(ch)
 			assert.FatalError(t, err)
-
 			expErr := ConnectionErr(errors.Errorf("error doing http GET for url "+
 				"http://zap.internal/.well-known/acme-challenge/%s with status code 400", ch.getToken()))
 			baseClone := ch.clone()
@@ -984,6 +983,7 @@ func TestHTTP01Validate(t *testing.T) {
 					assert.Equals(t, tc.res.getCreated(), ch.getCreated())
 					assert.Equals(t, tc.res.getValidated(), ch.getValidated())
 					assert.Equals(t, tc.res.getError(), ch.getError())
+					assert.Equals(t, tc.res.getRetry(), ch.getRetry())
 				}
 			}
 		})