From f96cf27193032120ba727316c4057dffed0cbe48 Mon Sep 17 00:00:00 2001 From: Miek Gieben Date: Fri, 18 Aug 2017 14:45:20 +0100 Subject: [PATCH] mw/federation: add federation back as separate mw for k8s (#929) * mw/federaration This PR add the federation back as a middleware to keep it more contained from the main kubernetes code. It also makes parseRequest less import and pushes this functionlity down in the k.Entries. This minimizes (or tries to) the importance for the qtype in the query. In the end the qtype checking should only happen in ServeDNS - but for k8s this might proof difficult. Numerous other cleanup in code and kubernetes tests. * up test coverage --- core/dnsserver/zdirectives.go | 1 + core/zmiddleware.go | 1 + middleware.cfg | 17 +-- middleware/autopath/autopath.go | 8 +- middleware/federation/README.md | 43 ++++++ middleware/federation/federation.go | 140 +++++++++++++++++++ middleware/federation/federation_test.go | 81 +++++++++++ middleware/federation/kubernetes_api_test.go | 111 +++++++++++++++ middleware/federation/nonwriter.go | 22 +++ middleware/federation/setup.go | 81 +++++++++++ middleware/federation/setup_test.go | 60 ++++++++ middleware/kubernetes/federation.go | 43 ++++++ middleware/kubernetes/handler_test.go | 6 +- middleware/kubernetes/kubernetes.go | 130 ++++++++--------- middleware/kubernetes/kubernetes_test.go | 3 +- middleware/kubernetes/local.go | 40 ++++++ middleware/kubernetes/ns.go | 4 +- middleware/kubernetes/ns_test.go | 6 +- middleware/kubernetes/parse.go | 20 +-- middleware/kubernetes/parse_test.go | 9 +- middleware/kubernetes/reverse_test.go | 3 +- middleware/kubernetes/setup.go | 3 +- middleware/kubernetes/setup_reverse_test.go | 4 +- middleware/test/helpers.go | 12 +- test/kubernetes_test.go | 2 +- 25 files changed, 727 insertions(+), 123 deletions(-) create mode 100644 middleware/federation/README.md create mode 100644 middleware/federation/federation.go create mode 100644 middleware/federation/federation_test.go create mode 100644 middleware/federation/kubernetes_api_test.go create mode 100644 middleware/federation/nonwriter.go create mode 100644 middleware/federation/setup.go create mode 100644 middleware/federation/setup_test.go create mode 100644 middleware/kubernetes/federation.go create mode 100644 middleware/kubernetes/local.go diff --git a/core/dnsserver/zdirectives.go b/core/dnsserver/zdirectives.go index 9a7e8c40a..528da4261 100644 --- a/core/dnsserver/zdirectives.go +++ b/core/dnsserver/zdirectives.go @@ -30,6 +30,7 @@ var directives = []string{ "dnssec", "reverse", "hosts", + "federation", "kubernetes", "file", "auto", diff --git a/core/zmiddleware.go b/core/zmiddleware.go index b21bdf0d2..aa5361712 100644 --- a/core/zmiddleware.go +++ b/core/zmiddleware.go @@ -15,6 +15,7 @@ import ( _ "github.com/coredns/coredns/middleware/erratic" _ "github.com/coredns/coredns/middleware/errors" _ "github.com/coredns/coredns/middleware/etcd" + _ "github.com/coredns/coredns/middleware/federation" _ "github.com/coredns/coredns/middleware/file" _ "github.com/coredns/coredns/middleware/health" _ "github.com/coredns/coredns/middleware/hosts" diff --git a/middleware.cfg b/middleware.cfg index e97aafa47..ba2ef74ff 100644 --- a/middleware.cfg +++ b/middleware.cfg @@ -38,13 +38,14 @@ 160:dnssec:dnssec 170:reverse:reverse 180:hosts:hosts -190:kubernetes:kubernetes -200:file:file -210:auto:auto -220:secondary:secondary -230:etcd:etcd -240:proxy:proxy -250:erratic:erratic -260:whoami:whoami +190:federation:federation +200:kubernetes:kubernetes +210:file:file +220:auto:auto +230:secondary:secondary +240:etcd:etcd +250:proxy:proxy +260:erratic:erratic +270:whoami:whoami 500:startup:github.com/mholt/caddy/startupshutdown 510:shutdown:github.com/mholt/caddy/startupshutdown diff --git a/middleware/autopath/autopath.go b/middleware/autopath/autopath.go index d3a8c84d2..ec6bb674b 100644 --- a/middleware/autopath/autopath.go +++ b/middleware/autopath/autopath.go @@ -1,8 +1,6 @@ -package autopath - /* -Autopath is a hack; it shortcuts the client's search path resolution by performing -these lookups on the server... +Autopath package implement autopathing. This is a hack; it shortcuts the +client's search path resolution by performing these lookups on the server... The server has a copy (via AutoPathFunc) of the client's search path and on receiving a query it first establish if the suffix matches the FIRST configured @@ -31,6 +29,7 @@ func (m Middleware ) AutoPath(state request.Request) []string { return []string{"first", "second", "last", ""} } */ +package autopath import ( "log" @@ -108,7 +107,6 @@ func (a *AutoPath) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Ms if err != nil { // Return now - not sure if this is the best. We should also check if the write has happened. return rcode, err - } if i == 0 { firstReply = nw.Msg diff --git a/middleware/federation/README.md b/middleware/federation/README.md new file mode 100644 index 000000000..c22a909f5 --- /dev/null +++ b/middleware/federation/README.md @@ -0,0 +1,43 @@ +# federation + +The *federation* middleware enables +[federated](https://kubernetes.io/docs/tasks/federation/federation-service-discovery/) queries to be +resolved via the kubernetes middleware. + +Enabling *federation* without also having *kubernetes* is a noop. + +## Syntax + +~~~ +federation [ZONES...] { + NAME DOMAIN +~~~ + +* Each **NAME** and **DOMAIN** defines federation membership. One entry for each. A duplicate + **NAME** will silently overwrite any previous value. + +## Examples + +Here we handle all service requests in the `prod` and `stage` federations. + +~~~ txt +. { + kubernetes cluster.local + federation cluster.local { + prod prod.feddomain.com + staging staging.feddomain.com + } +} +~~~ + +Or slightly shorter: + +~~~ txt +cluster.local { + kubernetes + federation { + prod prod.feddomain.com + staging staging.feddomain.com + } +} +~~~ diff --git a/middleware/federation/federation.go b/middleware/federation/federation.go new file mode 100644 index 000000000..caf29e630 --- /dev/null +++ b/middleware/federation/federation.go @@ -0,0 +1,140 @@ +/* +Package federation implements kubernetes federation. It checks if the qname matches +a possible federation. If this is the case and the captured answer is an NXDOMAIN, +federation is performed. If this is not the case the original answer is returned. + +The federation label is always the 2nd to last once the zone is chopped of. For +instance "nginx.mynamespace.myfederation.svc.example.com" has "myfederation" as +the federation label. For federation to work we do a normal k8s lookup +*without* that label, if that comes back with NXDOMAIN or NODATA(??) we create +a federation record and return that. + +Federation is only useful in conjunction with the kubernetes middleware, without it is a noop. +*/ +package federation + +import ( + "strings" + + "github.com/coredns/coredns/middleware" + "github.com/coredns/coredns/middleware/etcd/msg" + "github.com/coredns/coredns/middleware/pkg/dnsutil" + "github.com/coredns/coredns/request" + + "github.com/miekg/dns" + "golang.org/x/net/context" +) + +// Federation contains the name to zone mapping used for federation in kubernetes. +type Federation struct { + f map[string]string + zones []string + + Next middleware.Handler + Federations Func +} + +// Func needs to be implemented by any middleware that implements +// federation. Right now this is only the kubernetes middleware. +type Func func(state request.Request, fname, fzone string) (msg.Service, error) + +// New returns a new federation. +func New() *Federation { + return &Federation{f: make(map[string]string)} +} + +// ServeDNS implements the middleware.Handle interface. +func (f *Federation) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) { + if f.Federations == nil { + return middleware.NextOrFailure(f.Name(), f.Next, ctx, w, r) + } + + state := request.Request{W: w, Req: r} + zone := middleware.Zones(f.zones).Matches(state.Name()) + if zone == "" { + return middleware.NextOrFailure(f.Name(), f.Next, ctx, w, r) + } + + state.Zone = zone + + // Remove the federation label from the qname to see if something exists. + without, label := f.isNameFederation(state.Name(), state.Zone) + if without == "" { + return middleware.NextOrFailure(f.Name(), f.Next, ctx, w, r) + } + + qname := r.Question[0].Name + r.Question[0].Name = without + state.Clear() + + // Start the next middleware, but with a nowriter, capture the result, if NXDOMAIN + // perform federation, otherwise just write the result. + nw := NewNonWriter(w) + ret, err := middleware.NextOrFailure(f.Name(), f.Next, ctx, nw, r) + + if !middleware.ClientWrite(ret) { + // something went wrong + return ret, err + } + + if m := nw.Msg; m.Rcode != dns.RcodeNameError { + // If positive answer we need to substitute the orinal qname in question and answer. + r.Question[0].Name = qname + for _, a := range m.Answer { + a.Header().Name = qname + } + + state.SizeAndDo(m) + m, _ = state.Scrub(m) + w.WriteMsg(m) + + return dns.RcodeSuccess, nil + } + + // Still here, we've seen NXDOMAIN and need to perform federation. + service, err := f.Federations(state, label, f.f[label]) // state references Req which has updated qname + if err != nil { + return dns.RcodeServerFailure, err + } + + r.Question[0].Name = qname + + m := new(dns.Msg) + m.SetReply(r) + m.Authoritative, m.RecursionAvailable, m.Compress = true, true, true + + m.Answer = []dns.RR{service.NewCNAME(state.QName(), service.Host)} + + state.SizeAndDo(m) + m, _ = state.Scrub(m) + w.WriteMsg(m) + + return dns.RcodeSuccess, nil +} + +// Name implements the middleware.Handle interface. +func (f *Federation) Name() string { return "federation" } + +// IsNameFederation checks the qname to see if it is a potential federation. The federation +// label is always the 2nd to last once the zone is chopped of. For instance +// "nginx.mynamespace.myfederation.svc.example.com" has "myfederation" as the federation label. +// IsNameFederation returns a new qname with the federation label and the label itself or two +// emtpy strings if there wasn't a hit. +func (f *Federation) isNameFederation(name, zone string) (string, string) { + base, _ := dnsutil.TrimZone(name, zone) + + // TODO(miek): dns.PrevLabel is better for memory, or dns.Split. + labels := dns.SplitDomainName(base) + ll := len(labels) + if ll < 2 { + return "", "" + } + + fed := labels[ll-2] + + if _, ok := f.f[fed]; ok { + without := strings.Join(labels[:ll-2], ".") + "." + labels[ll-1] + "." + zone + return without, fed + } + return "", "" +} diff --git a/middleware/federation/federation_test.go b/middleware/federation/federation_test.go new file mode 100644 index 000000000..6f27cabad --- /dev/null +++ b/middleware/federation/federation_test.go @@ -0,0 +1,81 @@ +package federation + +import ( + "testing" + + "github.com/coredns/coredns/middleware/kubernetes" + "github.com/coredns/coredns/middleware/pkg/dnsrecorder" + "github.com/coredns/coredns/middleware/test" + + "github.com/miekg/dns" + "golang.org/x/net/context" +) + +func TestIsNameFederation(t *testing.T) { + tests := []struct { + fed string + qname string + expectedZone string + }{ + {"prod", "nginx.mynamespace.prod.svc.example.com.", "nginx.mynamespace.svc.example.com."}, + {"prod", "nginx.mynamespace.staging.svc.example.com.", ""}, + {"prod", "nginx.mynamespace.example.com.", ""}, + {"prod", "example.com.", ""}, + {"prod", "com.", ""}, + } + + fed := New() + for i, tc := range tests { + fed.f[tc.fed] = "test-name" + if x, _ := fed.isNameFederation(tc.qname, "example.com."); x != tc.expectedZone { + t.Errorf("Test %d, failed to get zone, expected %s, got %s", i, tc.expectedZone, x) + } + } +} + +func TestFederationKubernetes(t *testing.T) { + tests := []test.Case{ + { + // service exists so we return the IP address associated with it. + Qname: "svc1.testns.prod.svc.cluster.local.", Qtype: dns.TypeA, + Rcode: dns.RcodeSuccess, + Answer: []dns.RR{ + test.A("svc1.testns.prod.svc.cluster.local. 303 IN A 10.0.0.1"), + }, + }, + { + // service does not exist, do the federation dance. + Qname: "svc0.testns.prod.svc.cluster.local.", Qtype: dns.TypeA, + Rcode: dns.RcodeSuccess, + Answer: []dns.RR{ + test.CNAME("svc0.testns.prod.svc.cluster.local. 303 IN CNAME svc0.testns.prod.svc.fd-az.fd-r.federal.example."), + }, + }, + } + + k := kubernetes.New([]string{"cluster.local."}) + k.APIConn = &APIConnFederationTest{} + + fed := New() + fed.zones = []string{"cluster.local."} + fed.Federations = k.Federations + fed.Next = k + fed.f = map[string]string{ + "prod": "federal.example.", + } + + ctx := context.TODO() + for i, tc := range tests { + m := tc.Msg() + + rec := dnsrecorder.New(&test.ResponseWriter{}) + _, err := fed.ServeDNS(ctx, rec, m) + if err != nil { + t.Errorf("Test %d, expected no error, got %v\n", i, err) + return + } + + resp := rec.Msg + test.SortAndCheck(t, resp, tc) + } +} diff --git a/middleware/federation/kubernetes_api_test.go b/middleware/federation/kubernetes_api_test.go new file mode 100644 index 000000000..9e7056e49 --- /dev/null +++ b/middleware/federation/kubernetes_api_test.go @@ -0,0 +1,111 @@ +package federation + +import ( + "github.com/coredns/coredns/middleware/kubernetes" + + "k8s.io/client-go/1.5/pkg/api" +) + +type APIConnFederationTest struct{} + +func (APIConnFederationTest) Run() { return } +func (APIConnFederationTest) Stop() error { return nil } + +func (APIConnFederationTest) PodIndex(string) []interface{} { + a := make([]interface{}, 1) + a[0] = &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Namespace: "podns", + }, + Status: api.PodStatus{ + PodIP: "10.240.0.1", // Remote IP set in test.ResponseWriter + }, + } + return a +} + +func (APIConnFederationTest) ServiceList() []*api.Service { + svcs := []*api.Service{ + { + ObjectMeta: api.ObjectMeta{ + Name: "svc1", + Namespace: "testns", + }, + Spec: api.ServiceSpec{ + ClusterIP: "10.0.0.1", + Ports: []api.ServicePort{{ + Name: "http", + Protocol: "tcp", + Port: 80, + }}, + }, + }, + { + ObjectMeta: api.ObjectMeta{ + Name: "hdls1", + Namespace: "testns", + }, + Spec: api.ServiceSpec{ + ClusterIP: api.ClusterIPNone, + }, + }, + { + ObjectMeta: api.ObjectMeta{ + Name: "external", + Namespace: "testns", + }, + Spec: api.ServiceSpec{ + ExternalName: "ext.interwebs.test", + Ports: []api.ServicePort{{ + Name: "http", + Protocol: "tcp", + Port: 80, + }}, + }, + }, + } + return svcs + +} + +func (APIConnFederationTest) EndpointsList() api.EndpointsList { + return api.EndpointsList{ + Items: []api.Endpoints{ + { + Subsets: []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{ + { + IP: "172.0.0.1", + Hostname: "ep1a", + }, + }, + Ports: []api.EndpointPort{ + { + Port: 80, + Protocol: "tcp", + Name: "http", + }, + }, + }, + }, + ObjectMeta: api.ObjectMeta{ + Name: "svc1", + Namespace: "testns", + }, + }, + }, + } +} + +func (APIConnFederationTest) GetNodeByName(name string) (api.Node, error) { + return api.Node{ + ObjectMeta: api.ObjectMeta{ + Name: "test.node.foo.bar", + Labels: map[string]string{ + kubernetes.LabelRegion: "fd-r", + kubernetes.LabelZone: "fd-az", + }, + }, + }, nil +} diff --git a/middleware/federation/nonwriter.go b/middleware/federation/nonwriter.go new file mode 100644 index 000000000..c60fb1075 --- /dev/null +++ b/middleware/federation/nonwriter.go @@ -0,0 +1,22 @@ +package federation + +import ( + "github.com/miekg/dns" +) + +// NonWriter is a type of ResponseWriter that captures the message, but never writes to the client. +type NonWriter struct { + dns.ResponseWriter + Msg *dns.Msg +} + +// NewNonWriter makes and returns a new NonWriter. +func NewNonWriter(w dns.ResponseWriter) *NonWriter { return &NonWriter{ResponseWriter: w} } + +// WriteMsg records the message, but doesn't write it itself. +func (r *NonWriter) WriteMsg(res *dns.Msg) error { + r.Msg = res + return nil +} + +func (r *NonWriter) Write(buf []byte) (int, error) { return len(buf), nil } diff --git a/middleware/federation/setup.go b/middleware/federation/setup.go new file mode 100644 index 000000000..9a60fcc3d --- /dev/null +++ b/middleware/federation/setup.go @@ -0,0 +1,81 @@ +package federation + +import ( + "fmt" + + "github.com/coredns/coredns/core/dnsserver" + "github.com/coredns/coredns/middleware" + "github.com/coredns/coredns/middleware/kubernetes" + "github.com/miekg/dns" + + "github.com/mholt/caddy" +) + +func init() { + caddy.RegisterPlugin("federation", caddy.Plugin{ + ServerType: "dns", + Action: setup, + }) +} + +func setup(c *caddy.Controller) error { + fed, err := federationParse(c) + if err != nil { + return middleware.Error("federation", err) + } + + // Do this in OnStartup, so all middleware has been initialized. + c.OnStartup(func() error { + m := dnsserver.GetConfig(c).GetHandler("kubernetes") + if m == nil { + return nil + } + if x, ok := m.(kubernetes.Kubernetes); ok { + fed.Federations = x.Federations + } + return nil + }) + + dnsserver.GetConfig(c).AddMiddleware(func(next middleware.Handler) middleware.Handler { + fed.Next = next + return nil + }) + + return nil +} + +func federationParse(c *caddy.Controller) (*Federation, error) { + fed := New() + + for c.Next() { + // federation [zones..] + origins := make([]string, len(c.ServerBlockKeys)) + copy(origins, c.ServerBlockKeys) + + for c.NextBlock() { + x := c.Val() + switch c.Val() { + default: + args := c.RemainingArgs() + if len(args) != 1 { + return fed, fmt.Errorf("need two arguments for federation: %q", args) + } + fed.f[x] = dns.Fqdn(args[0]) + } + } + + for i := range origins { + origins[i] = middleware.Host(origins[i]).Normalize() + } + + fed.zones = origins + + if len(fed.f) == 0 { + return fed, fmt.Errorf("at least one name to zone federation expected") + } + + return fed, nil + } + + return fed, nil +} diff --git a/middleware/federation/setup_test.go b/middleware/federation/setup_test.go new file mode 100644 index 000000000..b0d09419a --- /dev/null +++ b/middleware/federation/setup_test.go @@ -0,0 +1,60 @@ +package federation + +import ( + "testing" + + "github.com/mholt/caddy" +) + +func TestSetup(t *testing.T) { + tests := []struct { + input string + shouldErr bool + expectedLen int + expectedNameZone []string // contains only entry for now + }{ + {`federation { + prod prod.example.org + }`, false, 1, []string{"prod", "prod.example.org."}}, + + {`federation { + staging staging.example.org + prod prod.example.org + }`, false, 2, []string{"prod", "prod.example.org."}}, + {`federation { + staging staging.example.org + prod prod.example.org + }`, false, 2, []string{"staging", "staging.example.org."}}, + // errors + {`federation { + }`, true, 0, []string{}}, + {`federation { + staging + }`, true, 0, []string{}}, + } + for i, test := range tests { + c := caddy.NewTestController("dns", test.input) + fed, err := federationParse(c) + if test.shouldErr && err == nil { + t.Errorf("Test %v: Expected error but found nil", i) + continue + } else if !test.shouldErr && err != nil { + t.Errorf("Test %v: Expected no error but found error: %v", i, err) + continue + } + if test.shouldErr && err != nil { + continue + } + + if x := len(fed.f); x != test.expectedLen { + t.Errorf("Test %v: Expected map length of %d, got: %d", i, test.expectedLen, x) + } + if x, ok := fed.f[test.expectedNameZone[0]]; !ok { + t.Errorf("Test %v: Expected name for %s, got nothing", i, test.expectedNameZone[0]) + } else { + if x != test.expectedNameZone[1] { + t.Errorf("Test %v: Expected zone: %s, got %s", i, test.expectedNameZone[1], x) + } + } + } +} diff --git a/middleware/kubernetes/federation.go b/middleware/kubernetes/federation.go new file mode 100644 index 000000000..90f1cca39 --- /dev/null +++ b/middleware/kubernetes/federation.go @@ -0,0 +1,43 @@ +package kubernetes + +import ( + "strings" + + "github.com/coredns/coredns/middleware/etcd/msg" + "github.com/coredns/coredns/request" +) + +// The federation node.Labels keys used. +const ( + // TODO: Do not hardcode these labels. Pull them out of the API instead. + // + // We can get them via .... + // import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + // metav1.LabelZoneFailureDomain + // metav1.LabelZoneRegion + // + // But importing above breaks coredns with flag collision of 'log_dir' + + LabelZone = "failure-domain.beta.kubernetes.io/zone" + LabelRegion = "failure-domain.beta.kubernetes.io/region" +) + +// Federations is used from the federations middleware to return the service that should be +// returned as a CNAME for federation(s) to work. +func (k *Kubernetes) Federations(state request.Request, fname, fzone string) (msg.Service, error) { + nodeName := k.localNodeName() + node, err := k.APIConn.GetNodeByName(nodeName) + if err != nil { + return msg.Service{}, err + } + r, err := k.parseRequest(state) + + lz := node.Labels[LabelZone] + lr := node.Labels[LabelRegion] + + if r.endpoint == "" { + return msg.Service{Host: strings.Join([]string{r.service, r.namespace, fname, r.podOrSvc, lz, lr, fzone}, ".")}, nil + } + + return msg.Service{Host: strings.Join([]string{r.endpoint, r.service, r.namespace, fname, r.podOrSvc, lz, lr, fzone}, ".")}, nil +} diff --git a/middleware/kubernetes/handler_test.go b/middleware/kubernetes/handler_test.go index e8ef49999..88a892404 100644 --- a/middleware/kubernetes/handler_test.go +++ b/middleware/kubernetes/handler_test.go @@ -146,10 +146,8 @@ var podModeVerifiedCases = map[string](test.Case){ func TestServeDNS(t *testing.T) { - k := Kubernetes{Zones: []string{"cluster.local."}} - + k := New([]string{"cluster.local."}) k.APIConn = &APIConnServeTest{} - k.interfaceAddrsFunc = localPodIP k.Next = test.NextHandler(dns.RcodeSuccess, nil) ctx := context.TODO() @@ -166,7 +164,7 @@ func TestServeDNS(t *testing.T) { runServeDNSTests(ctx, t, podModeVerifiedCases, k) } -func runServeDNSTests(ctx context.Context, t *testing.T, dnsTestCases map[string](test.Case), k Kubernetes) { +func runServeDNSTests(ctx context.Context, t *testing.T, dnsTestCases map[string](test.Case), k *Kubernetes) { for testname, tc := range dnsTestCases { r := tc.Msg() diff --git a/middleware/kubernetes/kubernetes.go b/middleware/kubernetes/kubernetes.go index b9ea683ff..419017bad 100644 --- a/middleware/kubernetes/kubernetes.go +++ b/middleware/kubernetes/kubernetes.go @@ -51,6 +51,16 @@ type Kubernetes struct { autoPathSearch []string // Local search path from /etc/resolv.conf. Needed for autopath. } +// New returns a intialized Kubernetes. It default interfaceAddrFunc to return 127.0.0.1. All other +// values default to their zero value, primaryZoneIndex will thus point to the first zone. +func New(zones []string) *Kubernetes { + k := new(Kubernetes) + k.Zones = zones + k.interfaceAddrsFunc = func() net.IP { return net.ParseIP("127.0.0.1") } + + return k +} + const ( // PodModeDisabled is the default value where pod requests are ignored PodModeDisabled = "disabled" @@ -96,21 +106,21 @@ func (k *Kubernetes) Services(state request.Request, exact bool, opt middleware. // We're looking again at types, which we've already done in ServeDNS, but there are some types k8s just can't answer. switch state.QType() { + case dns.TypeTXT: // 1 label + zone, label must be "dns-version". - t, err := dnsutil.TrimZone(state.Name(), state.Zone) - if err != nil { - return nil, nil, err - } + t, _ := dnsutil.TrimZone(state.Name(), state.Zone) + segs := dns.SplitDomainName(t) if len(segs) != 1 { - return nil, nil, errors.New("servfail") + return nil, nil, fmt.Errorf("kubernetes: TXT query can only be for dns-version: %s", state.QName()) } if segs[0] != "dns-version" { - return nil, nil, errInvalidRequest + return nil, nil, nil } svc := msg.Service{Text: DNSSchemaVersion, TTL: 28800, Key: msg.Path(state.QName(), "coredns")} return []msg.Service{svc}, nil, nil + case dns.TypeNS: // We can only get here if the qname equal the zone, see ServeDNS in handler.go. ns := k.nsAddr() @@ -118,38 +128,30 @@ func (k *Kubernetes) Services(state request.Request, exact bool, opt middleware. return []msg.Service{svc}, nil, nil } - r, e := k.parseRequest(state) - if e != nil { - return nil, nil, e + if state.QType() == dns.TypeA && isDefaultNS(state.Name(), state.Zone) { + // If this is an A request for "ns.dns", respond with a "fake" record for coredns. + // SOA records always use this hardcoded name + ns := k.nsAddr() + svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")} + return []msg.Service{svc}, nil, nil } - switch state.QType() { - case dns.TypeA, dns.TypeAAAA, dns.TypeCNAME: - if state.Type() == "A" && isDefaultNS(state.Name(), r) { - // If this is an A request for "ns.dns", respond with a "fake" record for coredns. - // SOA records always use this hardcoded name - ns := k.nsAddr() - svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")} - return []msg.Service{svc}, nil, nil - } - s, e := k.Entries(r) - if state.QType() == dns.TypeAAAA { - // AAAA not implemented - return nil, nil, e - } - return s, nil, e // Haven't implemented debug queries yet. - case dns.TypeSRV: - s, e := k.Entries(r) - // SRV for external services is not yet implemented, so remove those records - noext := []msg.Service{} - for _, svc := range s { - if t, _ := svc.HostType(); t != dns.TypeCNAME { - noext = append(noext, svc) - } - } - return noext, nil, e + s, e := k.Entries(state) + + // SRV for external services is not yet implemented, so remove those records. + + if state.QType() != dns.TypeSRV { + return s, nil, e } - return nil, nil, nil + + internal := []msg.Service{} + for _, svc := range s { + if t, _ := svc.HostType(); t != dns.TypeCNAME { + internal = append(internal, svc) + } + } + + return internal, nil, e } // primaryZone will return the first non-reverse zone being handled by this middleware @@ -247,9 +249,11 @@ func (k *Kubernetes) getClientConfig() (*rest.Config, error) { if len(k.APIClientKey) > 0 { authinfo.ClientKey = k.APIClientKey } + overrides.ClusterInfo = clusterinfo overrides.AuthInfo = authinfo clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) + return clientConfig.ClientConfig() } @@ -263,7 +267,7 @@ func (k *Kubernetes) InitKubeCache() (err error) { kubeClient, err := kubernetes.NewForConfig(config) if err != nil { - return fmt.Errorf("failed to create kubernetes notification controller: %v", err) + return fmt.Errorf("failed to create kubernetes notification controller: %q", err) } if k.LabelSelector != nil { @@ -271,12 +275,12 @@ func (k *Kubernetes) InitKubeCache() (err error) { selector, err = unversionedapi.LabelSelectorAsSelector(k.LabelSelector) k.Selector = &selector if err != nil { - return fmt.Errorf("unable to create Selector for LabelSelector '%s'.Error was: %s", k.LabelSelector, err) + return fmt.Errorf("unable to create Selector for LabelSelector '%s': %q", k.LabelSelector, err) } } if k.LabelSelector != nil { - log.Printf("[INFO] Kubernetes middleware configured with the label selector '%s'. Only kubernetes objects matching this label selector will be exposed.", unversionedapi.FormatLabelSelector(k.LabelSelector)) + log.Printf("[INFO] Kubernetes has label selector '%s'. Only objects matching this label selector will be exposed.", unversionedapi.FormatLabelSelector(k.LabelSelector)) } opts := dnsControlOpts{ @@ -287,20 +291,22 @@ func (k *Kubernetes) InitKubeCache() (err error) { return err } -// Records not implemented, see Entries(). +// Records is not implemented. func (k *Kubernetes) Records(name string, exact bool) ([]msg.Service, error) { - return nil, fmt.Errorf("NOOP") + return nil, fmt.Errorf("not implemented") } -// Entries looks up services in kubernetes. If exact is true, it will lookup -// just this name. This is used when find matches when completing SRV lookups -// for instance. -func (k *Kubernetes) Entries(r recordRequest) ([]msg.Service, error) { - // Abort if the namespace does not contain a wildcard, and namespace is not published per CoreFile - // Case where namespace contains a wildcard is handled in Get(...) method. - if (!wildcard(r.namespace)) && (len(k.Namespaces) > 0) && (!dnsstrings.StringInSlice(r.namespace, k.Namespaces)) { +// Entries looks up services in kubernetes. +func (k *Kubernetes) Entries(state request.Request) ([]msg.Service, error) { + r, e := k.parseRequest(state) + if e != nil { + return nil, e + } + + if !k.namespaceExposed(r.namespace) { return nil, errNsNotExposed } + services, pods, err := k.get(r) if err != nil { return nil, err @@ -310,7 +316,6 @@ func (k *Kubernetes) Entries(r recordRequest) ([]msg.Service, error) { } records := k.getRecordsForK8sItems(services, pods, r) - return records, nil } @@ -432,6 +437,7 @@ func (k *Kubernetes) findServices(r recordRequest) ([]kService, error) { if !(match(r.namespace, svc.Namespace, nsWildcard) && match(r.service, svc.Name, serviceWildcard)) { continue } + // If namespace has a wildcard, filter results against Corefile namespace list. // (Namespaces without a wildcard were filtered before the call to this function.) if nsWildcard && (len(k.Namespaces) > 0) && (!dnsstrings.StringInSlice(svc.Namespace, k.Namespaces)) { @@ -529,28 +535,22 @@ func (k *Kubernetes) getServiceRecordForIP(ip, name string) []msg.Service { return nil } +// namespaceExposed returns true when the namespace is exposed. +func (k *Kubernetes) namespaceExposed(namespace string) bool { + // Abort if the namespace does not contain a wildcard, and namespace is + // not published per CoreFile Case where namespace contains a wildcard + // is handled in k.get(...) method. + if (!wildcard(namespace)) && (len(k.Namespaces) > 0) && (!dnsstrings.StringInSlice(namespace, k.Namespaces)) { + return false + } + return true +} + // wildcard checks whether s contains a wildcard value func wildcard(s string) bool { return (s == "*" || s == "any") } -func localPodIP() net.IP { - addrs, err := net.InterfaceAddrs() - if err != nil { - return nil - } - - for _, addr := range addrs { - ip, _, _ := net.ParseCIDR(addr.String()) - ip = ip.To4() - if ip == nil || ip.IsLoopback() { - continue - } - return ip - } - return nil -} - const ( // Svc is the DNS schema for kubernetes services Svc = "svc" diff --git a/middleware/kubernetes/kubernetes_test.go b/middleware/kubernetes/kubernetes_test.go index 7af7d10ec..7caaa25de 100644 --- a/middleware/kubernetes/kubernetes_test.go +++ b/middleware/kubernetes/kubernetes_test.go @@ -195,8 +195,7 @@ func (APIConnServiceTest) GetNodeByName(name string) (api.Node, error) { func TestServices(t *testing.T) { - k := Kubernetes{Zones: []string{"interwebs.test."}} - k.interfaceAddrsFunc = localPodIP + k := New([]string{"interwebs.test."}) k.APIConn = &APIConnServiceTest{} type svcAns struct { diff --git a/middleware/kubernetes/local.go b/middleware/kubernetes/local.go new file mode 100644 index 000000000..e5b7f1e0f --- /dev/null +++ b/middleware/kubernetes/local.go @@ -0,0 +1,40 @@ +package kubernetes + +import "net" + +func localPodIP() net.IP { + addrs, err := net.InterfaceAddrs() + if err != nil { + return nil + } + + for _, addr := range addrs { + ip, _, _ := net.ParseCIDR(addr.String()) + ip = ip.To4() + if ip == nil || ip.IsLoopback() { + continue + } + return ip + } + return nil +} + +func (k *Kubernetes) localNodeName() string { + localIP := k.interfaceAddrsFunc() + if localIP == nil { + return "" + } + + // Find endpoint matching localIP + endpointsList := k.APIConn.EndpointsList() + for _, ep := range endpointsList.Items { + for _, eps := range ep.Subsets { + for _, addr := range eps.Addresses { + if localIP.Equal(net.ParseIP(addr.IP)) { + return *addr.NodeName + } + } + } + } + return "" +} diff --git a/middleware/kubernetes/ns.go b/middleware/kubernetes/ns.go index 8556469c4..4cacc382f 100644 --- a/middleware/kubernetes/ns.go +++ b/middleware/kubernetes/ns.go @@ -8,8 +8,8 @@ import ( "k8s.io/client-go/1.5/pkg/api" ) -func isDefaultNS(name string, r recordRequest) bool { - return strings.Index(name, defaultNSName) == 0 && strings.Index(name, r.zone) == len(defaultNSName) +func isDefaultNS(name, zone string) bool { + return strings.Index(name, defaultNSName) == 0 && strings.Index(name, zone) == len(defaultNSName) } func (k *Kubernetes) nsAddr() *dns.A { diff --git a/middleware/kubernetes/ns_test.go b/middleware/kubernetes/ns_test.go index 7815ade7e..8e9e80c71 100644 --- a/middleware/kubernetes/ns_test.go +++ b/middleware/kubernetes/ns_test.go @@ -1,7 +1,6 @@ package kubernetes import ( - "net" "testing" "k8s.io/client-go/1.5/pkg/api" @@ -36,7 +35,7 @@ func (APIConnTest) EndpointsList() api.EndpointsList { { Addresses: []api.EndpointAddress{ { - IP: "172.0.40.10", + IP: "127.0.0.1", }, }, }, @@ -54,8 +53,7 @@ func (APIConnTest) GetNodeByName(name string) (api.Node, error) { return api.Nod func TestNsAddr(t *testing.T) { - k := Kubernetes{Zones: []string{"inter.webs.test"}} - k.interfaceAddrsFunc = func() net.IP { return net.ParseIP("172.0.40.10") } + k := New([]string{"inter.webs.test."}) k.APIConn = &APIConnTest{} cdr := k.nsAddr() diff --git a/middleware/kubernetes/parse.go b/middleware/kubernetes/parse.go index c582f904e..c7d614ec1 100644 --- a/middleware/kubernetes/parse.go +++ b/middleware/kubernetes/parse.go @@ -25,24 +25,17 @@ type recordRequest struct { // parseRequest parses the qname to find all the elements we need for querying k8s. func (k *Kubernetes) parseRequest(state request.Request) (r recordRequest, err error) { - // 3 Possible cases: TODO(chris): remove federations comments here. - // SRV Request: _port._protocol.service.namespace.[federation.]type.zone - // A Request (endpoint): endpoint.service.namespace.[federation.]type.zone - // A Request (service): service.namespace.[federation.]type.zone + // 3 Possible cases: + // o SRV Request: _port._protocol.service.namespace.type.zone + // o A Request (endpoint): endpoint.service.namespace.type.zone + // o A Request (service): service.namespace.type.zone + // Federations are handled in the federation middleware. base, _ := dnsutil.TrimZone(state.Name(), state.Zone) segs := dns.SplitDomainName(base) r.zone = state.Zone - if state.QType() == dns.TypeNS { - return r, nil - } - - if state.QType() == dns.TypeA && isDefaultNS(state.Name(), r) { - return r, nil - } - offset := 0 if state.QType() == dns.TypeSRV { // The kubernetes peer-finder expects queries with empty port and service to resolve @@ -99,8 +92,7 @@ func (k *Kubernetes) parseRequest(state request.Request) (r recordRequest, err e return r, errInvalidRequest } -// String return a string representation of r, it just returns all -// fields concatenated with dots. +// String return a string representation of r, it just returns all fields concatenated with dots. // This is mostly used in tests. func (r recordRequest) String() string { s := r.port diff --git a/middleware/kubernetes/parse_test.go b/middleware/kubernetes/parse_test.go index 951a47554..0a3d224e4 100644 --- a/middleware/kubernetes/parse_test.go +++ b/middleware/kubernetes/parse_test.go @@ -4,11 +4,12 @@ import ( "testing" "github.com/coredns/coredns/request" + "github.com/miekg/dns" ) func TestParseRequest(t *testing.T) { - k := Kubernetes{Zones: []string{zone}} + k := New([]string{zone}) tests := []struct { query string @@ -30,10 +31,6 @@ func TestParseRequest(t *testing.T) { "1-2-3-4.webs.mynamespace.svc.inter.webs.test.", dns.TypeA, "..1-2-3-4.webs.mynamespace.svc.intern.webs.tests.", }, - { - "inter.webs.test.", dns.TypeNS, - "......intern.webs.tests.", - }, } for i, tc := range tests { m := new(dns.Msg) @@ -52,7 +49,7 @@ func TestParseRequest(t *testing.T) { } func TestParseInvalidRequest(t *testing.T) { - k := Kubernetes{Zones: []string{zone}} + k := New([]string{zone}) invalid := map[string]uint16{ "_http._tcp.webs.mynamespace.svc.inter.webs.test.": dns.TypeA, // A requests cannot have port or protocol diff --git a/middleware/kubernetes/reverse_test.go b/middleware/kubernetes/reverse_test.go index 759e615bd..aaf0907e8 100644 --- a/middleware/kubernetes/reverse_test.go +++ b/middleware/kubernetes/reverse_test.go @@ -77,8 +77,7 @@ func (APIConnReverseTest) GetNodeByName(name string) (api.Node, error) { func TestReverse(t *testing.T) { - k := Kubernetes{Zones: []string{"cluster.local.", "0.10.in-addr.arpa."}} - k.interfaceAddrsFunc = localPodIP + k := New([]string{"cluster.local.", "0.10.in-addr.arpa."}) k.APIConn = &APIConnReverseTest{} tests := []test.Case{ diff --git a/middleware/kubernetes/setup.go b/middleware/kubernetes/setup.go index e3a9093c3..123aa8001 100644 --- a/middleware/kubernetes/setup.go +++ b/middleware/kubernetes/setup.go @@ -67,10 +67,9 @@ func kubernetesParse(c *caddy.Controller) (*Kubernetes, error) { interfaceAddrsFunc: localPodIP, PodMode: PodModeDisabled, Proxy: proxy.Proxy{}, + autoPathSearch: searchFromResolvConf(), } - k8s.autoPathSearch = searchFromResolvConf() - for c.Next() { zones := c.RemainingArgs() diff --git a/middleware/kubernetes/setup_reverse_test.go b/middleware/kubernetes/setup_reverse_test.go index b6fa26b86..198bac0a5 100644 --- a/middleware/kubernetes/setup_reverse_test.go +++ b/middleware/kubernetes/setup_reverse_test.go @@ -19,12 +19,12 @@ func TestKubernetesParseReverseZone(t *testing.T) { c := caddy.NewTestController("dns", tc.input) k, err := kubernetesParse(c) if err != nil { - t.Fatalf("Test %d: Expected no error, got %q", err) + t.Fatalf("Test %d: Expected no error, got %q", i, err) } zl := len(k.Zones) if zl != len(tc.expectedZones) { - t.Errorf("Test %d: Expected kubernetes controller to be initialized with %d zones, found %d zones: '%v'", i, len(tc.expectedZones), zl) + t.Errorf("Test %d: Expected kubernetes to be initialized with %d zones, found %d zones", i, len(tc.expectedZones), zl) } for i, z := range tc.expectedZones { if k.Zones[i] != z { diff --git a/middleware/test/helpers.go b/middleware/test/helpers.go index 8d9cada43..2074f46c2 100644 --- a/middleware/test/helpers.go +++ b/middleware/test/helpers.go @@ -218,35 +218,35 @@ func Section(t *testing.T, tc Case, sec sect, rr []dns.RR) bool { case *dns.SOA: tt := section[i].(*dns.SOA) if x.Ns != tt.Ns { - t.Errorf("SOA nameserver should be %q, but is %q", x.Ns, tt.Ns) + t.Errorf("SOA nameserver should be %q, but is %q", tt.Ns, x.Ns) return false } case *dns.PTR: tt := section[i].(*dns.PTR) if x.Ptr != tt.Ptr { - t.Errorf("PTR ptr should be %q, but is %q", x.Ptr, tt.Ptr) + t.Errorf("PTR ptr should be %q, but is %q", tt.Ptr, x.Ptr) return false } case *dns.CNAME: tt := section[i].(*dns.CNAME) if x.Target != tt.Target { - t.Errorf("CNAME target should be %q, but is %q", x.Target, tt.Target) + t.Errorf("CNAME target should be %q, but is %q", tt.Target, x.Target) return false } case *dns.MX: tt := section[i].(*dns.MX) if x.Mx != tt.Mx { - t.Errorf("MX Mx should be %q, but is %q", x.Mx, tt.Mx) + t.Errorf("MX Mx should be %q, but is %q", tt.Mx, x.Mx) return false } if x.Preference != tt.Preference { - t.Errorf("MX Preference should be %q, but is %q", x.Preference, tt.Preference) + t.Errorf("MX Preference should be %q, but is %q", tt.Preference, x.Preference) return false } case *dns.NS: tt := section[i].(*dns.NS) if x.Ns != tt.Ns { - t.Errorf("NS nameserver should be %q, but is %q", x.Ns, tt.Ns) + t.Errorf("NS nameserver should be %q, but is %q", tt.Ns, x.Ns) return false } case *dns.OPT: diff --git a/test/kubernetes_test.go b/test/kubernetes_test.go index f9a0b00c4..2ee5a2d7f 100644 --- a/test/kubernetes_test.go +++ b/test/kubernetes_test.go @@ -357,7 +357,7 @@ func doIntegrationTests(t *testing.T, corefile string, testCases []test.Case) { } if len(res.Answer) != len(tc.Answer) { - t.Errorf("Expected %d answers but got %d for query %s, %d", len(tc.Answer), len(res.Answer), tc.Qname, tc.Qtype) + t.Errorf("Expected %d answers but got %d for query %s, %d\nfull reply %s", len(tc.Answer), len(res.Answer), tc.Qname, tc.Qtype, res) } //TODO: Check the actual RR values