gcloud: fix for wildcard (#740)

Gopkg.lock

@@ -682,6 +682,7 @@
     "golang.org/x/oauth2/clientcredentials",
     "golang.org/x/oauth2/google",
     "google.golang.org/api/dns/v1",
+    "google.golang.org/api/googleapi",
     "gopkg.in/ns1/ns1-go.v2/rest",
     "gopkg.in/ns1/ns1-go.v2/rest/model/dns",
     "gopkg.in/square/go-jose.v2",

challenge/dns01/dns_challenge.go

@@ -123,7 +123,7 @@ func (c *Challenge) Solve(authz acme.Authorization) error {
 
 	log.Infof("[%s] acme: Checking DNS record propagation using %+v", domain, recursiveNameservers)
 
-	err = wait.For(timeout, interval, func() (bool, error) {
+	err = wait.For("propagation", timeout, interval, func() (bool, error) {
 		stop, errP := c.preCheck.call(fqdn, value)
 		if !stop || errP != nil {
 			log.Infof("[%s] acme: Waiting for DNS record propagation.", domain)

challenge/dns01/precheck.go

@@ -91,10 +91,14 @@ func checkAuthoritativeNss(fqdn, value string, nameservers []string) (bool, erro
 			return false, fmt.Errorf("NS %s returned %s for %s", ns, dns.RcodeToString[r.Rcode], fqdn)
 		}
 
+		var records []string
+
 		var found bool
 		for _, rr := range r.Answer {
 			if txt, ok := rr.(*dns.TXT); ok {
-				if strings.Join(txt.Txt, "") == value {
+				record := strings.Join(txt.Txt, "")
+				records = append(records, record)
+				if record == value {
 					found = true
 					break
 				}
@@ -102,7 +106,7 @@ func checkAuthoritativeNss(fqdn, value string, nameservers []string) (bool, erro
 		}
 
 		if !found {
-			return false, fmt.Errorf("NS %s did not return the expected TXT record [fqdn: %s]", ns, fqdn)
+			return false, fmt.Errorf("NS %s did not return the expected TXT record [fqdn: %s, value: %s]: %s", ns, fqdn, value, strings.Join(records, " ,"))
 		}
 	}
 

e2e/loader/loader.go

@@ -141,7 +141,7 @@ func (l *EnvLoader) cmdPebble() (*exec.Cmd, *bytes.Buffer) {
 
 func pebbleHealthCheck(options *CmdOption) {
 	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
-	err := wait.For(10*time.Second, 500*time.Millisecond, func() (bool, error) {
+	err := wait.For("pebble", 10*time.Second, 500*time.Millisecond, func() (bool, error) {
 		resp, err := client.Get(options.HealthCheckURL)
 		if err != nil {
 			return false, err

platform/wait/wait.go

@@ -8,8 +8,8 @@ import (
 )
 
 // For polls the given function 'f', once every 'interval', up to 'timeout'.
-func For(timeout, interval time.Duration, f func() (bool, error)) error {
+func For(msg string, timeout, interval time.Duration, f func() (bool, error)) error {
-	log.Infof("Wait [timeout: %s, interval: %s]", timeout, interval)
+	log.Infof("Wait for %s [timeout: %s, interval: %s]", msg, timeout, interval)
 
 	var lastErr string
 	timeUp := time.After(timeout)

platform/wait/wait_test.go

@@ -8,7 +8,7 @@ import (
 func TestForTimeout(t *testing.T) {
 	c := make(chan error)
 	go func() {
-		err := For(3*time.Second, 1*time.Second, func() (bool, error) {
+		err := For("", 3*time.Second, 1*time.Second, func() (bool, error) {
 			return false, nil
 		})
 		c <- err

providers/dns/gcloud/googlecloud.go

@@ -8,17 +8,22 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/xenolf/lego/challenge/dns01"
+	"github.com/xenolf/lego/log"
 	"github.com/xenolf/lego/platform/config/env"
+	"github.com/xenolf/lego/platform/wait"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/dns/v1"
+	"google.golang.org/api/googleapi"
 )
 
 // Config is used to configure the creation of the DNSProvider
 type Config struct {
+	Debug              bool
 	Project            string
 	PropagationTimeout time.Duration
 	PollingInterval    time.Duration
@@ -29,6 +34,7 @@ type Config struct {
 // NewDefaultConfig returns a default configuration for the DNSProvider
 func NewDefaultConfig() *Config {
 	return &Config{
+		Debug:              env.GetOrDefaultBool("GCE_DEBUG", false),
 		TTL:                env.GetOrDefaultInt("GCE_TTL", dns01.DefaultTTL),
 		PropagationTimeout: env.GetOrDefaultSecond("GCE_PROPAGATION_TIMEOUT", 180*time.Second),
 		PollingInterval:    env.GetOrDefaultSecond("GCE_POLLING_INTERVAL", 5*time.Second),
@@ -131,11 +137,32 @@ func (d *DNSProvider) Present(domain, token, keyAuth string) error {
 	}
 
 	// Look for existing records.
-	existing, err := d.findTxtRecords(zone, fqdn)
+	existingRrSet, err := d.findTxtRecords(zone, fqdn)
 	if err != nil {
 		return fmt.Errorf("googlecloud: %v", err)
 	}
 
+	for _, rrSet := range existingRrSet {
+		var rrd []string
+		for _, rr := range rrSet.Rrdatas {
+			data := mustUnquote(rr)
+			rrd = append(rrd, data)
+
+			if data == value {
+				log.Printf("skip: the record already exists: %s", value)
+				return nil
+			}
+		}
+		rrSet.Rrdatas = rrd
+	}
+
+	// Attempt to delete the existing records before adding the new one.
+	if len(existingRrSet) > 0 {
+		if err = d.applyChanges(zone, &dns.Change{Deletions: existingRrSet}); err != nil {
+			return fmt.Errorf("googlecloud: %v", err)
+		}
+	}
+
 	rec := &dns.ResourceRecordSet{
 		Name:    fqdn,
 		Rrdatas: []string{value},
@@ -143,38 +170,71 @@ func (d *DNSProvider) Present(domain, token, keyAuth string) error {
 		Type:    "TXT",
 	}
 
-	change := &dns.Change{}
-
-	if len(existing) > 0 {
-		// Attempt to delete the existing records when adding our new one.
-		change.Deletions = existing
-
 	// Append existing TXT record data to the new TXT record data
-		for _, value := range existing {
+	for _, rrSet := range existingRrSet {
-			rec.Rrdatas = append(rec.Rrdatas, value.Rrdatas...)
+		for _, rr := range rrSet.Rrdatas {
+			if rr != value {
+				rec.Rrdatas = append(rec.Rrdatas, rrSet.Rrdatas...)
+			}
 		}
 	}
 
-	change.Additions = []*dns.ResourceRecordSet{rec}
+	change := &dns.Change{
+		Additions: []*dns.ResourceRecordSet{rec},
+	}
 
-	chg, err := d.client.Changes.Create(d.config.Project, zone, change).Do()
+	if err = d.applyChanges(zone, change); err != nil {
-	if err != nil {
 		return fmt.Errorf("googlecloud: %v", err)
 	}
 
-	// wait for change to be acknowledged
-	for chg.Status == "pending" {
-		time.Sleep(time.Second)
-
-		chg, err = d.client.Changes.Get(d.config.Project, zone, chg.Id).Do()
-		if err != nil {
-			return fmt.Errorf("googlecloud: %v", err)
-		}
-	}
-
 	return nil
 }
 
+func (d *DNSProvider) applyChanges(zone string, change *dns.Change) error {
+	if d.config.Debug {
+		data, _ := json.Marshal(change)
+		log.Printf("change (Create): %s", string(data))
+	}
+
+	chg, err := d.client.Changes.Create(d.config.Project, zone, change).Do()
+	if err != nil {
+		if v, ok := err.(*googleapi.Error); ok {
+			if v.Code == http.StatusNotFound {
+				return nil
+			}
+		}
+
+		data, _ := json.Marshal(change)
+		return fmt.Errorf("failed to perform changes [zone %s, change %s]: %v", zone, string(data), err)
+	}
+
+	if chg.Status == "done" {
+		return nil
+	}
+
+	chgID := chg.Id
+
+	// wait for change to be acknowledged
+	return wait.For("apply change", 30*time.Second, 3*time.Second, func() (bool, error) {
+		if d.config.Debug {
+			data, _ := json.Marshal(change)
+			log.Printf("change (Get): %s", string(data))
+		}
+
+		chg, err = d.client.Changes.Get(d.config.Project, zone, chgID).Do()
+		if err != nil {
+			data, _ := json.Marshal(change)
+			return false, fmt.Errorf("failed to get changes [zone %s, change %s]: %v", zone, string(data), err)
+		}
+
+		if chg.Status == "done" {
+			return true, nil
+		}
+
+		return false, fmt.Errorf("status: %s", chg.Status)
+	})
+}
+
 // CleanUp removes the TXT record matching the specified parameters.
 func (d *DNSProvider) CleanUp(domain, token, keyAuth string) error {
 	fqdn, _ := dns01.GetRecord(domain, keyAuth)
@@ -236,3 +296,11 @@ func (d *DNSProvider) findTxtRecords(zone, fqdn string) ([]*dns.ResourceRecordSe
 
 	return recs.Rrsets, nil
 }
+
+func mustUnquote(raw string) string {
+	clean, err := strconv.Unquote(raw)
+	if err != nil {
+		return raw
+	}
+	return clean
+}

providers/dns/nifcloud/nifcloud.go

@@ -146,7 +146,7 @@ func (d *DNSProvider) changeRecord(action, fqdn, value, domain string, ttl int)
 
 	statusID := resp.ChangeInfo.ID
 
-	return wait.For(120*time.Second, 4*time.Second, func() (bool, error) {
+	return wait.For("nifcloud", 120*time.Second, 4*time.Second, func() (bool, error) {
 		resp, err := d.client.GetChange(statusID)
 		if err != nil {
 			return false, fmt.Errorf("failed to query NIFCLOUD DNS change status: %v", err)

providers/dns/route53/route53.go

@@ -197,7 +197,7 @@ func (d *DNSProvider) changeRecord(action, hostedZoneID string, recordSet *route
 
 	changeID := resp.ChangeInfo.Id
 
-	return wait.For(d.config.PropagationTimeout, d.config.PollingInterval, func() (bool, error) {
+	return wait.For("route53", d.config.PropagationTimeout, d.config.PollingInterval, func() (bool, error) {
 		reqParams := &route53.GetChangeInput{Id: changeID}
 
 		resp, err := d.client.GetChange(reqParams)