plugin/kubernetes: Handle multiple local IPs and bind (#3208)

* use all local IPs

* mult/bind ips

* gofmt + boundIPs fix

* fix no matching endpoint case

* don't duplicate NS records in answer

* fix answer dedup

* fix comment

* add multi local ip test case

plugin/backend_lookup.go

@@ -372,6 +372,8 @@ func NS(ctx context.Context, b ServiceBackend, zone string, state request.Reques
 	// ... and reset
 	state.Req.Question[0].Name = old
 
+	seen := map[string]bool{}
+
 	for _, serv := range services {
 		what, ip := serv.HostType()
 		switch what {
@@ -380,8 +382,13 @@ func NS(ctx context.Context, b ServiceBackend, zone string, state request.Reques
 
 		case dns.TypeA, dns.TypeAAAA:
 			serv.Host = msg.Domain(serv.Key)
-			records = append(records, serv.NewNS(state.QName()))
 			extra = append(extra, newAddress(serv, serv.Host, ip, what))
+			ns := serv.NewNS(state.QName())
+			if _, ok := seen[ns.Ns]; ok {
+				continue
+			}
+			seen[ns.Ns] = true
+			records = append(records, ns)
 		}
 	}
 	return records, extra, nil

plugin/kubernetes/kubernetes.go

@@ -43,11 +43,10 @@ type Kubernetes struct {
 	Fall             fall.F
 	ttl              uint32
 	opts             dnsControlOpts
-
+	primaryZoneIndex int
-	primaryZoneIndex   int
+	localIPs         []net.IP
-	interfaceAddrsFunc func() net.IP
+	autoPathSearch   []string // Local search path from /etc/resolv.conf. Needed for autopath.
-	autoPathSearch     []string // Local search path from /etc/resolv.conf. Needed for autopath.
+	TransferTo       []string
-	TransferTo         []string
 }
 
 // New returns a initialized Kubernetes. It default interfaceAddrFunc to return 127.0.0.1. All other
@@ -56,7 +55,6 @@ func New(zones []string) *Kubernetes {
 	k := new(Kubernetes)
 	k.Zones = zones
 	k.Namespaces = make(map[string]struct{})
-	k.interfaceAddrsFunc = func() net.IP { return net.ParseIP("127.0.0.1") }
 	k.podMode = podModeDisabled
 	k.ttl = defaultTTL
 

plugin/kubernetes/kubernetes_apex_test.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"context"
+	"net"
 	"testing"
 
 	"github.com/coredns/coredns/plugin/pkg/dnstest"
@@ -63,6 +64,7 @@ func TestServeDNSApex(t *testing.T) {
 	k := New([]string{"cluster.local."})
 	k.APIConn = &APIConnServeTest{}
 	k.Next = test.NextHandler(dns.RcodeSuccess, nil)
+	k.localIPs = []net.IP{net.ParseIP("127.0.0.1")}
 	ctx := context.TODO()
 
 	for i, tc := range kubeApexCases {
@@ -85,7 +87,7 @@ func TestServeDNSApex(t *testing.T) {
 		}
 
 		if err := test.SortAndCheck(resp, tc); err != nil {
-			t.Error(err)
+			t.Errorf("Test %d: %v", i, err)
 		}
 	}
 }

plugin/kubernetes/kubernetes_test.go

@@ -310,20 +310,28 @@ func TestServicesAuthority(t *testing.T) {
 		key  string
 	}
 	type svcTest struct {
-		interfaceAddrs func() net.IP
+		localIPs []net.IP
-		qname          string
+		qname    string
-		qtype          uint16
+		qtype    uint16
-		answer         *svcAns
+		answer   []svcAns
 	}
 	tests := []svcTest{
-		{interfaceAddrs: func() net.IP { return net.ParseIP("127.0.0.1") }, qname: "ns.dns.interwebs.test.", qtype: dns.TypeA, answer: &svcAns{host: "127.0.0.1", key: "/" + coredns + "/test/interwebs/dns/ns"}},
+		{localIPs: []net.IP{net.ParseIP("1.2.3.4")}, qname: "ns.dns.interwebs.test.", qtype: dns.TypeA, answer: []svcAns{{host: "1.2.3.4", key: "/" + coredns + "/test/interwebs/dns/ns"}}},
-		{interfaceAddrs: func() net.IP { return net.ParseIP("127.0.0.1") }, qname: "ns.dns.interwebs.test.", qtype: dns.TypeAAAA},
+		{localIPs: []net.IP{net.ParseIP("1.2.3.4")}, qname: "ns.dns.interwebs.test.", qtype: dns.TypeAAAA},
-		{interfaceAddrs: func() net.IP { return net.ParseIP("::1") }, qname: "ns.dns.interwebs.test.", qtype: dns.TypeA},
+		{localIPs: []net.IP{net.ParseIP("1:2::3:4")}, qname: "ns.dns.interwebs.test.", qtype: dns.TypeA},
-		{interfaceAddrs: func() net.IP { return net.ParseIP("::1") }, qname: "ns.dns.interwebs.test.", qtype: dns.TypeAAAA, answer: &svcAns{host: "::1", key: "/" + coredns + "/test/interwebs/dns/ns"}},
+		{localIPs: []net.IP{net.ParseIP("1:2::3:4")}, qname: "ns.dns.interwebs.test.", qtype: dns.TypeAAAA, answer: []svcAns{{host: "1:2::3:4", key: "/" + coredns + "/test/interwebs/dns/ns"}}},
+		{
+			localIPs: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("1:2::3:4")},
+			qname:    "ns.dns.interwebs.test.",
+			qtype:    dns.TypeNS, answer: []svcAns{
+				{host: "1.2.3.4", key: "/" + coredns + "/test/interwebs/dns/ns"},
+				{host: "1:2::3:4", key: "/" + coredns + "/test/interwebs/dns/ns"},
+			},
+		},
 	}
 
 	for i, test := range tests {
-		k.interfaceAddrsFunc = test.interfaceAddrs
+		k.localIPs = test.localIPs
 
 		state := request.Request{
 			Req:  &dns.Msg{Question: []dns.Question{{Name: test.qname, Qtype: test.qtype}}},
@@ -334,7 +342,7 @@ func TestServicesAuthority(t *testing.T) {
 			t.Errorf("Test %d: got error '%v'", i, e)
 			continue
 		}
-		if test.answer != nil && len(svcs) != 1 {
+		if test.answer != nil && len(svcs) != len(test.answer) {
 			t.Errorf("Test %d, expected 1 answer, got %v", i, len(svcs))
 			continue
 		}
@@ -347,11 +355,13 @@ func TestServicesAuthority(t *testing.T) {
 			continue
 		}
 
-		if test.answer.host != svcs[0].Host {
+		for i, answer := range test.answer {
-			t.Errorf("Test %d, expected host '%v', got '%v'", i, test.answer.host, svcs[0].Host)
+			if answer.host != svcs[i].Host {
-		}
+				t.Errorf("Test %d, expected host '%v', got '%v'", i, answer.host, svcs[i].Host)
-		if test.answer.key != svcs[0].Key {
+			}
-			t.Errorf("Test %d, expected key '%v', got '%v'", i, test.answer.key, svcs[0].Key)
+			if answer.key != svcs[i].Key {
+				t.Errorf("Test %d, expected key '%v', got '%v'", i, answer.key, svcs[i].Key)
+			}
 		}
 	}
 }

plugin/kubernetes/local.go

@@ -2,41 +2,54 @@ package kubernetes
 
 import (
 	"net"
+
+	"github.com/caddyserver/caddy"
+	"github.com/coredns/coredns/core/dnsserver"
 )
 
-func localPodIP() net.IP {
+// boundIPs returns the list of non-loopback IPs that CoreDNS is bound to
-	addrs, err := net.InterfaceAddrs()
+func boundIPs(c *caddy.Controller) (ips []net.IP) {
-	if err != nil {
+	conf := dnsserver.GetConfig(c)
-		return nil
+	hosts := conf.ListenHosts
+	if hosts == nil || hosts[0] == "" {
+		hosts = nil
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			return nil
+		}
+		for _, addr := range addrs {
+			hosts = append(hosts, addr.String())
+		}
 	}
-
+	for _, host := range hosts {
-	for _, addr := range addrs {
+		ip, _, _ := net.ParseCIDR(host)
-		ip, _, _ := net.ParseCIDR(addr.String())
 		ip4 := ip.To4()
 		if ip4 != nil && !ip4.IsLoopback() {
-			return ip4
+			ips = append(ips, ip4)
+			continue
 		}
 		ip6 := ip.To16()
 		if ip6 != nil && !ip6.IsLoopback() {
-			return ip6
+			ips = append(ips, ip6)
 		}
 	}
-	return nil
+	return ips
 }
 
 // LocalNodeName is exclusively used in federation plugin, will be deprecated later.
 func (k *Kubernetes) LocalNodeName() string {
-	localIP := k.interfaceAddrsFunc()
+	if len(k.localIPs) == 0 {
-	if localIP == nil {
 		return ""
 	}
 
-	// Find endpoint matching localIP
+	// Find fist endpoint matching any localIP
-	for _, ep := range k.APIConn.EpIndexReverse(localIP.String()) {
+	for _, localIP := range k.localIPs {
-		for _, eps := range ep.Subsets {
+		for _, ep := range k.APIConn.EpIndexReverse(localIP.String()) {
-			for _, addr := range eps.Addresses {
+			for _, eps := range ep.Subsets {
-				if localIP.Equal(net.ParseIP(addr.IP)) {
+				for _, addr := range eps.Addresses {
-					return addr.NodeName
+					if localIP.Equal(net.ParseIP(addr.IP)) {
+						return addr.NodeName
+					}
 				}
 			}
 		}

plugin/kubernetes/ns.go

@@ -21,15 +21,10 @@ func (k *Kubernetes) nsAddrs(external bool, zone string) []dns.RR {
 		svcIPs   []net.IP
 	)
 
-	// Find the CoreDNS Endpoint
+	// Find the CoreDNS Endpoints
-	localIP := k.interfaceAddrsFunc()
+	for _, localIP := range k.localIPs {
-	endpoints := k.APIConn.EpIndexReverse(localIP.String())
+		endpoints := k.APIConn.EpIndexReverse(localIP.String())
 
-	// If the CoreDNS Endpoint is not found, use the locally bound IP address
-	if len(endpoints) == 0 {
-		svcNames = []string{defaultNSName + zone}
-		svcIPs = []net.IP{localIP}
-	} else {
 		// Collect IPs for all Services of the Endpoints
 		for _, endpoint := range endpoints {
 			svcs := k.APIConn.SvcIndex(object.ServiceKey(endpoint.Name, endpoint.Namespace))
@@ -59,6 +54,16 @@ func (k *Kubernetes) nsAddrs(external bool, zone string) []dns.RR {
 		}
 	}
 
+	// If no local IPs matched any endpoints, use the localIPs directly
+	if len(svcIPs) == 0 {
+		svcIPs = make([]net.IP, len(k.localIPs))
+		svcNames = make([]string, len(k.localIPs))
+		for i, localIP := range k.localIPs {
+			svcNames[i] = defaultNSName + zone
+			svcIPs[i] = localIP
+		}
+	}
+
 	// Create an RR slice of collected IPs
 	var rrs []dns.RR
 	rrs = make([]dns.RR, len(svcIPs))

plugin/kubernetes/ns_test.go

@@ -111,7 +111,7 @@ func TestNsAddrs(t *testing.T) {
 
 	k := New([]string{"inter.webs.test."})
 	k.APIConn = &APIConnTest{}
-	k.interfaceAddrsFunc = func() net.IP { return net.ParseIP("10.244.0.20") }
+	k.localIPs = []net.IP{net.ParseIP("10.244.0.20")}
 
 	cdrs := k.nsAddrs(false, k.Zones[0])
 

plugin/kubernetes/setup.go

@@ -61,6 +61,12 @@ func setup(c *caddy.Controller) error {
 		return k
 	})
 
+	// get locally bound addresses
+	c.OnStartup(func() error {
+		k.localIPs = boundIPs(c)
+		return nil
+	})
+
 	return nil
 }
 
@@ -113,7 +119,6 @@ func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
 func ParseStanza(c *caddy.Controller) (*Kubernetes, error) {
 
 	k8s := New([]string{""})
-	k8s.interfaceAddrsFunc = localPodIP
 	k8s.autoPathSearch = searchFromResolvConf()
 
 	opts := dnsControlOpts{