middleware/kubernetes: Server side path lookups (#750)

* initial commit

* add config options

* add readme

* rewording

* revert unlreated change

* normalize host domain path

* add ndots opt, allow > 1 host domains, pull host domains from resolv.conf

* implementing review feedback

* update readme

* use dns lib, config format, defaults

* Correct autopath example.
This commit is contained in:
Chris O'Haver 2017-06-28 18:44:30 -04:00 committed by John Belamaric
parent 817f3960b8
commit edf71fb168
6 changed files with 407 additions and 51 deletions

View file

@ -121,6 +121,58 @@ kubernetes coredns.local {
# Each line consists of the name of the federation, and the domain.
federation myfed foo.example.com
# autopath [NDOTS [RESPONSE [RESOLV-CONF]]
#
# Enables server side search path lookups for pods. When enabled, coredns
# will identify search path queries from pods and perform the remaining
# lookups in the path on the pod's behalf. The search path used mimics the
# resolv.conf search path deployed to pods. E.g.
#
# search ns1.svc.cluster.local svc.cluster.local cluster.local foo.com
#
# If no domains in the path produce an answer, a lookup on the bare question
# will be attempted.
#
# A successful response will contain a question section with the original
# question, and an answer section containing the record for the question that
# actually had an answer. This means that the question and answer will not
# match. For example:
#
# # host -v -t a google.com
# Trying "google.com.default.svc.cluster.local"
# ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 50957
# ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
#
# ;; QUESTION SECTION:
# ;google.com.default.svc.cluster.local. IN A
#
# ;; ANSWER SECTION:
# google.com. 175 IN A 216.58.194.206
#
#
# NDOTS (default: 0) This provides an adjustable threshold to
# prevent server side lookups from triggering. If the number of dots before
# the first search domain is less than this number, then the search path will
# not executed on the server side.
#
# RESPONSE (default: SERVFAIL) RESPONSE can be either NXDOMAIN, SERVFAIL or
# NOERROR. This option causes coredns to return the given response instead of
# NXDOMAIN when the all searches in the path produce no results. Setting this
# to SERVFAIL or NOERROR should prevent the client from fruitlessly continuing
# the client side searches in the path after the server already checked them.
#
# RESOLV-CONF (default: /etc/resolv.conf) If specified, coredns uses this
# file to get the host's search domains. CoreDNS performs a lookup on these
# domains if the in-cluster search domains in the path fail to produce an
# answer. If not specified, the values will be read from the local resolv.conf
# file (i.e the resolv.conf file in the pod containing coredns).
#
# Enabling autopath causes coredns to use more memory since it needs to
# maintain a watch on all pods. If autopath and "pods verified" mode are
# both enabled, they will share the same watch. I.e. enabling both options
# should have an equivalent memory impact of just one.
autopath 0 SERVFAIL /etc/resolv.conf
# fallthrough
#
# If a query for a record in the cluster zone results in NXDOMAIN,

View file

@ -2,9 +2,11 @@ package kubernetes
import (
"errors"
"strings"
"github.com/coredns/coredns/middleware"
"github.com/coredns/coredns/middleware/pkg/dnsutil"
"github.com/coredns/coredns/middleware/rewrite"
"github.com/coredns/coredns/request"
"github.com/miekg/dns"
@ -39,37 +41,55 @@ func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.M
zone = state.Name()
}
var (
records, extra []dns.RR
err error
)
switch state.Type() {
case "A":
records, _, err = middleware.A(&k, zone, state, nil, middleware.Options{})
case "AAAA":
records, _, err = middleware.AAAA(&k, zone, state, nil, middleware.Options{})
case "TXT":
records, _, err = middleware.TXT(&k, zone, state, middleware.Options{})
case "CNAME":
records, _, err = middleware.CNAME(&k, zone, state, middleware.Options{})
case "PTR":
records, _, err = middleware.PTR(&k, zone, state, middleware.Options{})
case "MX":
records, extra, _, err = middleware.MX(&k, zone, state, middleware.Options{})
case "SRV":
records, extra, _, err = middleware.SRV(&k, zone, state, middleware.Options{})
case "SOA":
records, _, err = middleware.SOA(&k, zone, state, middleware.Options{})
case "NS":
if state.Name() == zone {
records, extra, _, err = middleware.NS(&k, zone, state, middleware.Options{})
break
records, extra, _, err := k.routeRequest(zone, state)
if k.AutoPath.Enabled && k.IsNameError(err) {
p := k.findPodWithIP(state.IP())
for p != nil {
name, path, ok := splitSearch(zone, state.QName(), p.Namespace)
if !ok {
break
}
if (dns.CountLabel(name) - 1) < k.AutoPath.NDots {
break
}
// Search "svc.cluster.local" and "cluster.local"
for i := 0; i < 2; i++ {
path = strings.Join(dns.SplitDomainName(path)[1:], ".")
state = state.NewWithQuestion(strings.Join([]string{name, path}, "."), state.QType())
records, extra, _, err = k.routeRequest(zone, state)
if !k.IsNameError(err) {
break
}
}
if !k.IsNameError(err) {
break
}
// Fallthrough with the host search path (if set)
wr := rewrite.NewResponseReverter(w, r)
for _, hostsearch := range k.AutoPath.HostSearchPath {
r = state.NewWithQuestion(strings.Join([]string{name, hostsearch}, "."), state.QType()).Req
rcode, nextErr := middleware.NextOrFailure(k.Name(), k.Next, ctx, wr, r)
if rcode == dns.RcodeSuccess {
return rcode, nextErr
}
}
// Search . in this middleware
state = state.NewWithQuestion(strings.Join([]string{name, "."}, ""), state.QType())
records, extra, _, err = k.routeRequest(zone, state)
if !k.IsNameError(err) {
break
}
// Search . in the next middleware
r = state.Req
rcode, nextErr := middleware.NextOrFailure(k.Name(), k.Next, ctx, wr, r)
if rcode == dns.RcodeNameError {
rcode = k.AutoPath.OnNXDOMAIN
}
return rcode, nextErr
}
fallthrough
default:
// Do a fake A lookup, so we can distinguish between NODATA and NXDOMAIN
_, _, err = middleware.A(&k, zone, state, nil, middleware.Options{})
}
if k.IsNameError(err) {
if k.Fallthrough {
return middleware.NextOrFailure(k.Name(), k.Next, ctx, w, r)
@ -95,5 +115,36 @@ func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.M
return dns.RcodeSuccess, nil
}
func (k *Kubernetes) routeRequest(zone string, state request.Request) (records []dns.RR, extra []dns.RR, debug []dns.RR, err error) {
switch state.Type() {
case "A":
records, _, err = middleware.A(k, zone, state, nil, middleware.Options{})
case "AAAA":
records, _, err = middleware.AAAA(k, zone, state, nil, middleware.Options{})
case "TXT":
records, _, err = middleware.TXT(k, zone, state, middleware.Options{})
case "CNAME":
records, _, err = middleware.CNAME(k, zone, state, middleware.Options{})
case "PTR":
records, _, err = middleware.PTR(k, zone, state, middleware.Options{})
case "MX":
records, extra, _, err = middleware.MX(k, zone, state, middleware.Options{})
case "SRV":
records, extra, _, err = middleware.SRV(k, zone, state, middleware.Options{})
case "SOA":
records, _, err = middleware.SOA(k, zone, state, middleware.Options{})
case "NS":
if state.Name() == zone {
records, extra, _, err = middleware.NS(k, zone, state, middleware.Options{})
break
}
fallthrough
default:
// Do a fake A lookup, so we can distinguish between NODATA and NXDOMAIN
_, _, err = middleware.A(k, zone, state, nil, middleware.Options{})
}
return records, extra, nil, err
}
// Name implements the Handler interface.
func (k Kubernetes) Name() string { return "kubernetes" }

View file

@ -28,26 +28,35 @@ import (
// Kubernetes implements a middleware that connects to a Kubernetes cluster.
type Kubernetes struct {
Next middleware.Handler
Zones []string
primaryZone int
Proxy proxy.Proxy // Proxy for looking up names during the resolution process
APIEndpoint string
APICertAuth string
APIClientCert string
APIClientKey string
APIConn dnsController
ResyncPeriod time.Duration
Namespaces []string
Federations []Federation
LabelSelector *unversionedapi.LabelSelector
Selector *labels.Selector
PodMode string
ReverseCidrs []net.IPNet
Fallthrough bool
Next middleware.Handler
Zones []string
primaryZone int
Proxy proxy.Proxy // Proxy for looking up names during the resolution process
APIEndpoint string
APICertAuth string
APIClientCert string
APIClientKey string
APIConn dnsController
ResyncPeriod time.Duration
Namespaces []string
Federations []Federation
LabelSelector *unversionedapi.LabelSelector
Selector *labels.Selector
PodMode string
ReverseCidrs []net.IPNet
Fallthrough bool
AutoPath
interfaceAddrs interfaceAddrser
}
type AutoPath struct {
Enabled bool
NDots int
ResolvConfFile string
HostSearchPath []string
OnNXDOMAIN int
}
const (
// PodModeDisabled is the default value where pod requests are ignored
PodModeDisabled = "disabled"
@ -97,6 +106,7 @@ var errInvalidRequest = errors.New("invalid query name")
var errZoneNotFound = errors.New("zone not found")
var errAPIBadPodType = errors.New("expected type *api.Pod")
var errPodsDisabled = errors.New("pod records disabled")
var errResolvConfReadErr = errors.New("resolv.conf read error")
// Services implements the ServiceBackend interface.
func (k *Kubernetes) Services(state request.Request, exact bool, opt middleware.Options) (svcs []msg.Service, debug []msg.Service, err error) {
@ -183,7 +193,7 @@ func (k *Kubernetes) Lookup(state request.Request, name string, typ uint16) (*dn
// IsNameError implements the ServiceBackend interface.
func (k *Kubernetes) IsNameError(err error) bool {
return err == errNoItems || err == errNsNotExposed || err == errInvalidRequest
return err == errNoItems || err == errNsNotExposed || err == errInvalidRequest || err == errZoneNotFound
}
// Debug implements the ServiceBackend interface.
@ -245,7 +255,7 @@ func (k *Kubernetes) InitKubeCache() (err error) {
}
opts := dnsControlOpts{
initPodCache: k.PodMode == PodModeVerified,
initPodCache: (k.PodMode == PodModeVerified || k.AutoPath.Enabled),
}
k.APIConn = newdnsController(kubeClient, k.ResyncPeriod, k.Selector, opts)
@ -448,6 +458,21 @@ func ipFromPodName(podname string) string {
return strings.Replace(podname, "-", ":", -1)
}
func (k *Kubernetes) findPodWithIP(ip string) (p *api.Pod) {
if k.PodMode != PodModeVerified {
return nil
}
objList := k.APIConn.PodIndex(ip)
for _, o := range objList {
p, ok := o.(*api.Pod)
if !ok {
return nil
}
return p
}
return nil
}
func (k *Kubernetes) findPods(namespace, podname string) (pods []pod, err error) {
if k.PodMode == PodModeDisabled {
return pods, errPodsDisabled
@ -634,3 +659,11 @@ func (k *Kubernetes) localPodIP() net.IP {
}
return nil
}
func splitSearch(zone, question, namespace string) (name, search string, ok bool) {
search = strings.Join([]string{namespace, "svc", zone}, ".")
if dns.IsSubDomain(search, question) {
return question[:len(question)-len(search)-1], search, true
}
return "", "", false
}

View file

@ -480,3 +480,27 @@ func TestServices(t *testing.T) {
}
}
func TestSplitSearchPath(t *testing.T) {
type testCase struct {
question string
namespace string
expectedName string
expectedSearch string
expectedOk bool
}
tests := []testCase{
{question: "test.blah.com", namespace: "ns1", expectedName: "", expectedSearch: "", expectedOk: false},
{question: "foo.com.ns2.svc.interwebs.nets", namespace: "ns1", expectedName: "", expectedSearch: "", expectedOk: false},
{question: "foo.com.svc.interwebs.nets", namespace: "ns1", expectedName: "", expectedSearch: "", expectedOk: false},
{question: "foo.com.ns1.svc.interwebs.nets", namespace: "ns1", expectedName: "foo.com", expectedSearch: "ns1.svc.interwebs.nets", expectedOk: true},
}
zone := "interwebs.nets"
for _, c := range tests {
name, search, ok := splitSearch(zone, c.question, c.namespace)
if c.expectedName != name || c.expectedSearch != search || c.expectedOk != ok {
t.Errorf("Case %v: Expected name'%v', search:'%v', ok:'%v'. Got name:'%v', search:'%v', ok:'%v'.", c.question, c.expectedName, c.expectedSearch, c.expectedOk, name, search, ok)
}
}
}

View file

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
@ -11,6 +12,7 @@ import (
"github.com/coredns/coredns/middleware"
"github.com/coredns/coredns/middleware/pkg/dnsutil"
"github.com/coredns/coredns/middleware/proxy"
"github.com/miekg/dns"
"github.com/mholt/caddy"
unversionedapi "k8s.io/client-go/1.5/pkg/api/unversioned"
@ -187,7 +189,48 @@ func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
continue
}
return nil, fmt.Errorf("incorrect number of arguments for federation, got %v, expected 2", len(args))
case "autopath": // name zone
args := c.RemainingArgs()
k8s.AutoPath = AutoPath{
NDots: defautNdots,
HostSearchPath: []string{},
ResolvConfFile: defaultResolvConfFile,
OnNXDOMAIN: defaultOnNXDOMAIN,
}
if len(args) > 3 {
return nil, fmt.Errorf("incorrect number of arguments for autopath, got %v, expected at most 3", len(args))
}
if len(args) > 0 {
ndots, err := strconv.Atoi(args[0])
if err != nil {
return nil, fmt.Errorf("invalid NDOTS argument for autopath, got '%v', expected an integer", ndots)
}
k8s.AutoPath.NDots = ndots
}
if len(args) > 1 {
switch args[1] {
case dns.RcodeToString[dns.RcodeNameError]:
k8s.AutoPath.OnNXDOMAIN = dns.RcodeNameError
case dns.RcodeToString[dns.RcodeSuccess]:
k8s.AutoPath.OnNXDOMAIN = dns.RcodeSuccess
case dns.RcodeToString[dns.RcodeServerFailure]:
k8s.AutoPath.OnNXDOMAIN = dns.RcodeServerFailure
default:
return nil, fmt.Errorf("invalid RESPONSE argument for autopath, got '%v', expected SERVFAIL, NOERROR, or NXDOMAIN", args[1])
}
}
if len(args) > 2 {
k8s.AutoPath.ResolvConfFile = args[2]
}
rc, err := dns.ClientConfigFromFile(k8s.AutoPath.ResolvConfFile)
if err != nil {
return nil, fmt.Errorf("error when parsing %v: %v", k8s.AutoPath.ResolvConfFile, err)
}
k8s.AutoPath.HostSearchPath = rc.Search
middleware.Zones(k8s.AutoPath.HostSearchPath).Normalize()
k8s.AutoPath.Enabled = true
continue
}
}
return k8s, nil
@ -197,6 +240,9 @@ func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) {
}
const (
defaultResyncPeriod = 5 * time.Minute
defaultPodMode = PodModeDisabled
defaultResyncPeriod = 5 * time.Minute
defaultPodMode = PodModeDisabled
defautNdots = 0
defaultResolvConfFile = "/etc/resolv.conf"
defaultOnNXDOMAIN = dns.RcodeServerFailure
)

View file

@ -2,11 +2,16 @@ package kubernetes
import (
"net"
"os"
"reflect"
"strings"
"testing"
"time"
"github.com/coredns/coredns/middleware/test"
"github.com/mholt/caddy"
"github.com/miekg/dns"
unversionedapi "k8s.io/client-go/1.5/pkg/api/unversioned"
)
@ -16,6 +21,13 @@ func parseCidr(cidr string) net.IPNet {
}
func TestKubernetesParse(t *testing.T) {
f, rm, err := test.TempFile(os.TempDir(), testResolveConf)
autoPathResolvConfFile := f
if err != nil {
t.Fatalf("Could not create resolv.conf TempFile: %s", err)
}
defer rm()
tests := []struct {
description string // Human-facing description of test case
input string // Corefile data as string
@ -30,6 +42,7 @@ func TestKubernetesParse(t *testing.T) {
expectedFallthrough bool
expectedUpstreams []string
expectedFederations []Federation
expectedAutoPath AutoPath
}{
// positive
{
@ -46,6 +59,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"kubernetes keyword with multiple zones",
@ -61,6 +75,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"kubernetes keyword with zone and empty braces",
@ -77,6 +92,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"endpoint keyword with url",
@ -94,6 +110,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"namespaces keyword with one namespace",
@ -111,6 +128,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
nil,
AutoPath{},
},
{
"namespaces keyword with multiple namespaces",
@ -128,6 +146,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"resync period in seconds",
@ -145,6 +164,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"resync period in minutes",
@ -162,6 +182,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"basic label selector",
@ -179,6 +200,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"multi-label selector",
@ -196,6 +218,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"fully specified valid config",
@ -217,6 +240,7 @@ func TestKubernetesParse(t *testing.T) {
true,
nil,
[]Federation{},
AutoPath{},
},
// negative
{
@ -233,6 +257,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"kubernetes keyword without a zone",
@ -248,6 +273,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"endpoint keyword without an endpoint value",
@ -265,6 +291,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"namespace keyword without a namespace value",
@ -282,6 +309,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"resyncperiod keyword without a duration value",
@ -299,6 +327,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"resync period no units",
@ -316,6 +345,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"resync period invalid",
@ -333,6 +363,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"labels with no selector value",
@ -350,6 +381,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
{
"labels with invalid selector value",
@ -367,6 +399,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// pods disabled
{
@ -385,6 +418,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// pods insecure
{
@ -403,6 +437,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// pods verified
{
@ -421,6 +456,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// pods invalid
{
@ -439,6 +475,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// cidrs ok
{
@ -457,6 +494,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// cidrs ok
{
@ -475,6 +513,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// fallthrough invalid
{
@ -493,6 +532,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// Valid upstream
{
@ -511,6 +551,7 @@ func TestKubernetesParse(t *testing.T) {
false,
[]string{"13.14.15.16:53"},
[]Federation{},
AutoPath{},
},
// Invalid upstream
{
@ -529,6 +570,7 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// Valid federations
{
@ -551,6 +593,7 @@ func TestKubernetesParse(t *testing.T) {
{name: "foo", zone: "bar.crawl.com"},
{name: "fed", zone: "era.tion.com"},
},
AutoPath{},
},
// Invalid federations
{
@ -569,6 +612,104 @@ func TestKubernetesParse(t *testing.T) {
false,
nil,
[]Federation{},
AutoPath{},
},
// autopath
{
"valid autopath",
`kubernetes coredns.local {
autopath 1 NXDOMAIN ` + autoPathResolvConfFile + `
}`,
false,
"",
1,
0,
defaultResyncPeriod,
"",
defaultPodMode,
nil,
false,
nil,
nil,
AutoPath{
Enabled: true,
NDots: 1,
HostSearchPath: []string{"bar.com.", "baz.com."},
ResolvConfFile: autoPathResolvConfFile,
OnNXDOMAIN: dns.RcodeNameError,
},
},
{
"invalid autopath RESPONSE",
`kubernetes coredns.local {
autopath 0 CRY
}`,
true,
"invalid RESPONSE argument for autopath",
-1,
0,
defaultResyncPeriod,
"",
defaultPodMode,
nil,
false,
nil,
nil,
AutoPath{},
},
{
"invalid autopath NDOTS",
`kubernetes coredns.local {
autopath polka
}`,
true,
"invalid NDOTS argument for autopath",
-1,
0,
defaultResyncPeriod,
"",
defaultPodMode,
nil,
false,
nil,
nil,
AutoPath{},
},
{
"invalid autopath RESOLV-CONF",
`kubernetes coredns.local {
autopath 1 NOERROR /wrong/path/to/resolv.conf
}`,
true,
"error when parsing",
-1,
0,
defaultResyncPeriod,
"",
defaultPodMode,
nil,
false,
nil,
nil,
AutoPath{},
},
{
"invalid autopath invalid option",
`kubernetes coredns.local {
autopath 1 SERVFAIL ` + autoPathResolvConfFile + ` foo
}`,
true,
"incorrect number of arguments",
-1,
0,
defaultResyncPeriod,
"",
defaultPodMode,
nil,
false,
nil,
nil,
AutoPath{},
},
}
@ -669,6 +810,15 @@ func TestKubernetesParse(t *testing.T) {
}
}
// autopath
if !reflect.DeepEqual(test.expectedAutoPath, k8sController.AutoPath) {
t.Errorf("Test %d: Expected kubernetes controller to be initialized with autopath '%v'. Instead found autopath '%v' for input '%s'", i, test.expectedAutoPath, k8sController.AutoPath, test.input)
}
}
}
const testResolveConf = `nameserver 1.2.3.4
domain foo.com
search bar.com baz.com
options ndots:5
`