diff --git a/.gitignore b/.gitignore index 362016d3b..fe121cbce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ query.log Corefile *.swp coredns -conf/devk8sCorefile diff --git a/Makefile b/Makefile index eb433f7b5..faedc63c8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -#VERBOSE := -VERBOSE := -v +#BUILD_VERBOSE := +BUILD_VERBOSE := -v + +TEST_VERBOSE := +#TEST_VERBOSE := -v all: - go build $(VERBOSE) + go build $(BUILD_VERBOSE) .PHONY: docker docker: @@ -11,11 +14,11 @@ docker: .PHONY: deps deps: - go get + go get ${BUILD_VERBOSE} .PHONY: test test: - go test + go test $(TEST_VERBOSE) ./... .PHONY: clean clean: diff --git a/conf/k8sCorefile b/conf/k8sCorefile index 7825a626e..fc446e350 100644 --- a/conf/k8sCorefile +++ b/conf/k8sCorefile @@ -4,6 +4,10 @@ kubernetes coredns.local { # Use url for k8s API endpoint endpoint http://localhost:8080 + # Assemble k8s record names with the template + template {service}.{namespace}.{zone} + # Only expose the k8s namespace "demo" + namespaces demo } # Perform DNS response caching for the coredns.local zone # Cache timeout is provided by the integer in seconds diff --git a/core/setup/kubernetes.go b/core/setup/kubernetes.go index 8f21286f5..f6e4b4806 100644 --- a/core/setup/kubernetes.go +++ b/core/setup/kubernetes.go @@ -1,29 +1,34 @@ package setup import ( -// "crypto/tls" -// "crypto/x509" - "fmt" -// "io/ioutil" -// "net" -// "net/http" -// "time" + //"crypto/tls" + //"crypto/x509" + "fmt" + //"io/ioutil" + //"net" + //"net/http" + "strings" + //"time" "github.com/miekg/coredns/middleware" "github.com/miekg/coredns/middleware/kubernetes" k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" + "github.com/miekg/coredns/middleware/kubernetes/nametemplate" "github.com/miekg/coredns/middleware/proxy" -// "github.com/miekg/coredns/middleware/singleflight" + //"github.com/miekg/coredns/middleware/singleflight" "golang.org/x/net/context" ) -const defaultK8sEndpoint = "http://localhost:8080" +const ( + defaultK8sEndpoint = "http://localhost:8080" + defaultNameTemplate = "{service}.{namespace}.{zone}" +) // Kubernetes sets up the kubernetes middleware. func Kubernetes(c *Controller) (middleware.Middleware, error) { - fmt.Println("controller %v", c) - // TODO: Determine if subzone support required + fmt.Println("controller %v", c) + // TODO: Determine if subzone support required kubernetes, err := kubernetesParse(c) @@ -39,29 +44,40 @@ func Kubernetes(c *Controller) (middleware.Middleware, error) { func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) { - /* - * TODO: Remove unused state and simplify. - * Inflight and Ctx might not be needed. Leaving in place until - * we take a pass at API caching and optimizing connector to the - * k8s API. Single flight (or limited upper-bound) for inflight - * API calls may be desirable. - */ + /* + * TODO: Remove unused state and simplify. + * Inflight and Ctx might not be needed. Leaving in place until + * we take a pass at API caching and optimizing connector to the + * k8s API. Single flight (or limited upper-bound) for inflight + * API calls may be desirable. + */ k8s := kubernetes.Kubernetes{ - Proxy: proxy.New([]string{}), - Ctx: context.Background(), -// Inflight: &singleflight.Group{}, - APIConn: nil, + Proxy: proxy.New([]string{}), + Ctx: context.Background(), + // Inflight: &singleflight.Group{}, } var ( - endpoints = []string{defaultK8sEndpoint} + endpoints = []string{defaultK8sEndpoint} + template = defaultNameTemplate + namespaces = []string{} ) + + k8s.APIConn = k8sc.NewK8sConnector(endpoints[0]) + k8s.NameTemplate = new(nametemplate.NameTemplate) + k8s.NameTemplate.SetTemplate(template) + for c.Next() { if c.Val() == "kubernetes" { - k8s.Zones = c.RemainingArgs() - if len(k8s.Zones) == 0 { + zones := c.RemainingArgs() + + if len(zones) == 0 { k8s.Zones = c.ServerBlockHosts + } else { + // Normalize requested zones + k8s.Zones = kubernetes.NormalizeZoneList(zones) } + middleware.Zones(k8s.Zones).FullyQualify() if c.NextBlock() { // TODO(miek): 2 switches? @@ -72,22 +88,33 @@ func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) { return kubernetes.Kubernetes{}, c.ArgErr() } endpoints = args - k8s.APIConn = k8sc.NewK8sConnector(endpoints[0]) + k8s.APIConn = k8sc.NewK8sConnector(endpoints[0]) } for c.Next() { switch c.Val() { - case "endpoint": + case "template": args := c.RemainingArgs() if len(args) == 0 { return kubernetes.Kubernetes{}, c.ArgErr() } - endpoints = args + template = strings.Join(args, "") + err := k8s.NameTemplate.SetTemplate(template) + if err != nil { + return kubernetes.Kubernetes{}, err + } + case "namespaces": + args := c.RemainingArgs() + if len(args) == 0 { + return kubernetes.Kubernetes{}, c.ArgErr() + } + namespaces = args + k8s.Namespaces = &namespaces } } } return k8s, nil } - fmt.Println("endpoints='%v'", endpoints) } + fmt.Println("here before return") return kubernetes.Kubernetes{}, nil } diff --git a/middleware/kubernetes/README.md b/middleware/kubernetes/README.md index 7e2e42dc2..c5ba32f72 100644 --- a/middleware/kubernetes/README.md +++ b/middleware/kubernetes/README.md @@ -14,7 +14,7 @@ are constructed as "myservice.mynamespace.coredns.local" where: kubernetes [zones...] ~~~ -* `zones` zones kubernetes should be authorative for. +* `zones` zones kubernetes should be authorative for. Overlapping zones are ignored. ~~~ @@ -88,7 +88,7 @@ The kubernetes control client can be downloaded from the generic URL: `http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${GOOS}/${GOARCH}/${K8S_BINARY}` For example, the kubectl client for Linux can be downloaded using the command: -`curl -sSL "http://storage.googleapis.com/kubernetes-release/release/v1.2.4/bin/linux/amd64/kubectl" +`curl -sSL "http://storage.googleapis.com/kubernetes-release/release/v1.2.4/bin/linux/amd64/kubectl"` The following `setup_kubectl.sh` script can be stored in the same directory as kubectl to setup @@ -248,37 +248,100 @@ return the IP addresses for all services with "nginx" in the service name. TBD: * How does this relate the the k8s load-balancer configuration? * Do wildcards search across namespaces? -* Initial implementation assumes that a namespace maps to the first DNS label below the zone managed by the kubernetes middleware. This assumption may need to be revised. +* Initial implementation assumes that a namespace maps to the first DNS label + below the zone managed by the kubernetes middleware. This assumption may + need to be revised. ## TODO -* Implement namespace filtering to different zones. -* Implement IP selection and ordering (internal/external). -* Implement SRV-record queries using naive lookup. -* Flatten service and namespace names to valid DNS characters. (service names - and namespace names in k8s may use uppercase and non-DNS characters. Implement - flattening to lower case and mapping of non-DNS characters to DNS characters - in a standard way.) -* Do we need to generate synthetic zone records for namespaces? -* Implement wildcard-based lookup. -* Improve lookup to reduce size of query result obtained from k8s API. - (namespace-based?, other ideas?) -* How to support label specification in Corefile to allow use of labels to - indicate zone? (Is this even useful?) For example, the following configuration - exposes all services labeled for the "staging" environment and tenant "customerB" - in the zone "customerB.stage.local": +* SkyDNS compatibility/equivalency: + * Kubernetes packaging and execution + * Automate packaging to allow executing in Kubernetes. That is, add Docker + container build as target in Makefile. Also include anything else needed + to simplify launch as the k8s DNS service. + Note: Dockerfile already exists in coredns repo to build the docker image. + This work item should identify how to pass configuration and run as a SkyDNS + replacement. + * Identify any kubernetes changes necessary to use coredns as k8s DNS server. That is, + how do we consume the "--cluster-dns=" and "--cluster-domain=" arguments. + * Work out how to pass CoreDNS configuration via kubectl command line and yaml + service definition file. + * Ensure that resolver in each kubernetes container is configured to use + coredns instance. + * Update kubernetes middleware documentation to describe running CoreDNS as a + SkyDNS replacement. (Include descriptions of different ways to pass CoreFile + to coredns command.) + * Expose load-balancer IP addresses. + * Calculate SRV priority based on number of instances running. + (See SkyDNS README.md) + * Functional work + * Implement wildcard-based lookup. Minimally support `*`, consider `?` as well. + * Note from Miek on PR 181: "SkyDNS also supports the word `any`. + * Implement SkyDNS-style synthetic zones such as "svc" to group k8s objects. (This + should be optional behavior.) Also look at "pod" synthetic zones. + * Implement test cases for SkyDNS equivalent functionality. + * SkyDNS functionality, as listed in SkyDNS README: https://github.com/kubernetes/kubernetes/blob/release-1.2/cluster/addons/dns/README.md + * A records in form of `pod-ip-address.my-namespace.cluster.local`. + For example, a pod with ip `1.2.3.4` in the namespace `default` + with a dns name of `cluster.local` would have an entry: + `1-2-3-4.default.pod.cluster.local`. + * SRV records in form of + `_my-port-name._my-port-protocol.my-namespace.svc.cluster.local` + CNAME records for both regular services and headless services. + See SkyDNS README. + * A Records and hostname Based on Pod Annotations (k8s beta 1.2 feature). + See SkyDNS README. + * Note: the embedded IP and embedded port record names are weird. I + would need to know the IP/port in order to create the query to lookup + the name. Presumably these are intended for wildcard queries. + * Performance + * Improve lookup to reduce size of query result obtained from k8s API. + (namespace-based?, other ideas?) + * Caching of k8s API dataset. + * DNS response caching is good, but we should also cache at the http query + level as well. (Take a look at https://github.com/patrickmn/go-cache as + a potential expiring cache implementation for the http API queries.) + * Push notifications from k8s for data changes rather than pull via API? +* Additional features: + * Implement namespace filtering to different zones. That is, zone "a.b" + publishes services from namespace "foo", and zone "x.y" publishes services + from namespaces "bar" and "baz". (Basic version implemented -- need test cases.) + * Reverse IN-ADDR entries for services. (Is there any value in supporting + reverse lookup records? + * How to support label specification in Corefile to allow use of labels to + indicate zone? (Is this even useful?) For example, the following + configuration exposes all services labeled for the "staging" environment + and tenant "customerB" in the zone "customerB.stage.local": -~~~ -kubernetes customerB.stage.local { - # Use url for k8s API endpoint - endpoint http://localhost:8080 - label "environment" : "staging", "tenant" : "customerB" -} -~~~ - -* Test with CoreDNS caching. CoreDNS caching for DNS response is working using - the `cache` directive. Tested working using 20s cache timeout and A-record queries. -* DNS response caching is good, but we should also cache at the http query - level as well. (Take a look at https://github.com/patrickmn/go-cache as - a potential expiring cache implementation for the http API queries.) + kubernetes customerB.stage.local { + # Use url for k8s API endpoint + endpoint http://localhost:8080 + label "environment" : "staging", "tenant" : "customerB" + } + Note: label specification/selection is a killer feature for segmenting + test vs staging vs prod environments. + * Implement IP selection and ordering (internal/external). Related to + wildcards and SkyDNS use of CNAMES. + * Flatten service and namespace names to valid DNS characters. (service names + and namespace names in k8s may use uppercase and non-DNS characters. Implement + flattening to lower case and mapping of non-DNS characters to DNS characters + in a standard way.) + * Expose arbitrary kubernetes repository data as TXT records? + * Support custom user-provided templates for k8s names. A string provided + in the middleware configuration like `{service}.{namespace}.{type}` defines + the template of how to construct record names for the zone. This example + would produce `myservice.mynamespace.svc.cluster.local`. (Basic template + implemented. Need to slice zone out of current template implementation.) +* DNS Correctness + * Do we need to generate synthetic zone records for namespaces? + * Do we need to generate synthetic zone records for the skydns synthetic zones? +* Test cases + * ~~Implement test cases for http data parsing using dependency injection + for http get operations.~~ + * Test with CoreDNS caching. CoreDNS caching for DNS response is working + using the `cache` directive. Tested working using 20s cache timeout + and A-record queries. Automate testing with cache in place. + * Automate CoreDNS performance tests. Initially for zone files, and for + pre-loaded k8s API cache. + * Automate integration testing with kubernetes. diff --git a/middleware/kubernetes/SkyDNS.md b/middleware/kubernetes/SkyDNS.md new file mode 100644 index 000000000..5670d3d87 --- /dev/null +++ b/middleware/kubernetes/SkyDNS.md @@ -0,0 +1,44 @@ +## DNS Schema + +Notes about the SkyDNS record naming scheme. (Copied from SkyDNS project README for reference while +hacking on the k8s middleware.) + +### Services + +#### A Records + +"Normal" (not headless) Services are assigned a DNS A record for a name of the form `my-svc.my-namespace.svc.cluster.local.` +This resolves to the cluster IP of the Service. + +"Headless" (without a cluster IP) Services are also assigned a DNS A record for a name of the form `my-svc.my-namespace.svc.cluster.local.` +Unlike normal Services, this resolves to the set of IPs of the pods selected by the Service. +Clients are expected to consume the set or else use standard round-robin selection from the set. + + +### Pods + +#### A Records + +When enabled, pods are assigned a DNS A record in the form of `pod-ip-address.my-namespace.pod.cluster.local.` + +For example, a pod with ip `1.2.3.4` in the namespace default with a dns name of `cluster.local` would have +an entry: `1-2-3-4.default.pod.cluster.local.` + +####A Records and hostname Based on Pod Annotations - A Beta Feature in Kubernetes v1.2 +Currently when a pod is created, its hostname is the Pod's `metadata.name` value. +With v1.2, users can specify a Pod annotation, `pod.beta.kubernetes.io/hostname`, to specify what the Pod's hostname should be. +If the annotation is specified, the annotation value takes precendence over the Pod's name, to be the hostname of the pod. +For example, given a Pod with annotation `pod.beta.kubernetes.io/hostname: my-pod-name`, the Pod will have its hostname set to "my-pod-name". + +v1.2 introduces a beta feature where the user can specify a Pod annotation, `pod.beta.kubernetes.io/subdomain`, to specify what the Pod's subdomain should be. +If the annotation is specified, the fully qualified Pod hostname will be "...svc.". +For example, given a Pod with the hostname annotation set to "foo", and the subdomain annotation set to "bar", in namespace "my-namespace", the pod will set its own FQDN as "foo.bar.my-namespace.svc.cluster.local" + +If there exists a headless service in the same namespace as the pod and with the same name as the subdomain, the cluster's KubeDNS Server will also return an A record for the Pod's fully qualified hostname. +Given a Pod with the hostname annotation set to "foo" and the subdomain annotation set to "bar", and a headless Service named "bar" in the same namespace, the pod will see it's own FQDN as "foo.bar.my-namespace.svc.cluster.local". DNS will serve an A record at that name, pointing to the Pod's IP. + +With v1.2, the Endpoints object also has a new annotation `endpoints.beta.kubernetes.io/hostnames-map`. Its value is the json representation of map[string(IP)][endpoints.HostRecord], for example: '{"10.245.1.6":{HostName: "my-webserver"}}'. +If the Endpoints are for a headless service, then A records will be created with the format ...svc. +For the example json, if endpoints are for a headless service named "bar", and one of the endpoints has IP "10.245.1.6", then a A record will be created with the name "my-webserver.bar.my-namespace.svc.cluster.local" and the A record lookup would return "10.245.1.6". +This endpoints annotation generally does not need to be specified by end-users, but can used by the internal service controller to deliver the aforementioned feature. + diff --git a/middleware/kubernetes/handler.go b/middleware/kubernetes/handler.go index 44de8da3d..168b65508 100644 --- a/middleware/kubernetes/handler.go +++ b/middleware/kubernetes/handler.go @@ -11,15 +11,15 @@ import ( func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) { - fmt.Println("[debug] here entering ServeDNS: ctx:%v dnsmsg:%v", ctx, r) + fmt.Printf("[debug] here entering ServeDNS: ctx:%v dnsmsg:%v\n", ctx, r) state := middleware.State{W: w, Req: r} if state.QClass() != dns.ClassINET { return dns.RcodeServerFailure, fmt.Errorf("can only deal with ClassINET") } - // Check that query matches one of the zones served by this middleware, - // otherwise delegate to the next in the pipeline. + // Check that query matches one of the zones served by this middleware, + // otherwise delegate to the next in the pipeline. zone := middleware.Zones(k.Zones).Matches(state.Name()) if zone == "" { if k.Next == nil { @@ -43,6 +43,9 @@ func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.M records, err = k.AAAA(zone, state, nil) case "TXT": records, err = k.TXT(zone, state) + // TODO: change lookup to return appropriate error. Then add code below + // this switch to check for the error and return not implemented. + //return dns.RcodeNotImplemented, nil case "CNAME": records, err = k.CNAME(zone, state) case "MX": diff --git a/middleware/kubernetes/k8sclient/dataobjects.go b/middleware/kubernetes/k8sclient/dataobjects.go index a5ab4f19c..b17adeba4 100644 --- a/middleware/kubernetes/k8sclient/dataobjects.go +++ b/middleware/kubernetes/k8sclient/dataobjects.go @@ -1,110 +1,113 @@ package k8sclient import ( - "encoding/json" - "net/http" + "encoding/json" + "net/http" ) - -func getJson(url string, target interface{}) error { - r, err := http.Get(url) - if err != nil { - return err - } - defer r.Body.Close() - - return json.NewDecoder(r.Body).Decode(target) +// getK8sAPIResponse wraps the http.Get(url) function to provide dependency +// injection for unit testing. +var getK8sAPIResponse = func(url string) (resp *http.Response, err error) { + resp, err = http.Get(url) + return resp, err } +func parseJson(url string, target interface{}) error { + r, err := getK8sAPIResponse(url) + if err != nil { + return err + } + defer r.Body.Close() + + return json.NewDecoder(r.Body).Decode(target) +} // Kubernetes Resource List type ResourceList struct { - Kind string `json:"kind"` - GroupVersion string `json:"groupVersion"` - Resources []resource `json:"resources"` + Kind string `json:"kind"` + GroupVersion string `json:"groupVersion"` + Resources []resource `json:"resources"` } type resource struct { - Name string `json:"name"` - Namespaced bool `json:"namespaced"` - Kind string `json:"kind"` + Name string `json:"name"` + Namespaced bool `json:"namespaced"` + Kind string `json:"kind"` } - // Kubernetes NamespaceList type NamespaceList struct { - Kind string `json:"kind"` - APIVersion string `json:"apiVersion"` - Metadata apiListMetadata `json:"metadata"` - Items []nsItems `json:"items"` + Kind string `json:"kind"` + APIVersion string `json:"apiVersion"` + Metadata apiListMetadata `json:"metadata"` + Items []nsItems `json:"items"` } type apiListMetadata struct { - SelfLink string `json:"selfLink"` - resourceVersion string `json:"resourceVersion"` + SelfLink string `json:"selfLink"` + ResourceVersion string `json:"resourceVersion"` } type nsItems struct { - Metadata nsMetadata `json:"metadata"` - Spec nsSpec `json:"spec"` - Status nsStatus `json:"status"` + Metadata nsMetadata `json:"metadata"` + Spec nsSpec `json:"spec"` + Status nsStatus `json:"status"` } type nsMetadata struct { - Name string `json:"name"` - SelfLink string `json:"selfLink"` - Uid string `json:"uid"` - ResourceVersion string `json:"resourceVersion"` - CreationTimestamp string `json:"creationTimestamp"` + Name string `json:"name"` + SelfLink string `json:"selfLink"` + Uid string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + CreationTimestamp string `json:"creationTimestamp"` } type nsSpec struct { - Finalizers []string `json:"finalizers"` + Finalizers []string `json:"finalizers"` } type nsStatus struct { - Phase string `json:"phase"` + Phase string `json:"phase"` } - // Kubernetes ServiceList type ServiceList struct { - Kind string `json:"kind"` - APIVersion string `json:"apiVersion"` - Metadata apiListMetadata `json:"metadata"` - Items []ServiceItem `json:"items"` + Kind string `json:"kind"` + APIVersion string `json:"apiVersion"` + Metadata apiListMetadata `json:"metadata"` + Items []ServiceItem `json:"items"` } type ServiceItem struct { - Metadata serviceMetadata `json:"metadata"` - Spec serviceSpec `json:"spec"` -// Status serviceStatus `json:"status"` + Metadata serviceMetadata `json:"metadata"` + Spec serviceSpec `json:"spec"` + // Status serviceStatus `json:"status"` } type serviceMetadata struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - SelfLink string `json:"selfLink"` - Uid string `json:"uid"` - ResourceVersion string `json:"resourceVersion"` - CreationTimestamp string `json:"creationTimestamp"` - // labels + Name string `json:"name"` + Namespace string `json:"namespace"` + SelfLink string `json:"selfLink"` + Uid string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + CreationTimestamp string `json:"creationTimestamp"` + // labels } type serviceSpec struct { - Ports []servicePort `json:"ports"` - ClusterIP string `json:"clusterIP"` - Type string `json:"type"` - SessionAffinity string `json:"sessionAffinity"` + Ports []servicePort `json:"ports"` + ClusterIP string `json:"clusterIP"` + Type string `json:"type"` + SessionAffinity string `json:"sessionAffinity"` } type servicePort struct { - Name string `json:"name"` - Protocol string `json:"protocol"` - Port int `json:"port"` - TargetPort int `json:"targetPort"` + Name string `json:"name"` + Protocol string `json:"protocol"` + Port int `json:"port"` + TargetPort int `json:"targetPort"` } type serviceStatus struct { - LoadBalancer string `json:"loadBalancer"` + LoadBalancer string `json:"loadBalancer"` } diff --git a/middleware/kubernetes/k8sclient/k8sclient.go b/middleware/kubernetes/k8sclient/k8sclient.go index a05ef8905..95300f3b9 100644 --- a/middleware/kubernetes/k8sclient/k8sclient.go +++ b/middleware/kubernetes/k8sclient/k8sclient.go @@ -1,117 +1,157 @@ package k8sclient import ( -// "fmt" - "net/url" + "errors" + "fmt" + "net/url" + "strings" ) // API strings const ( - apiBase = "/api/v1" - apiNamespaces = "/namespaces" - apiServices = "/services" + apiBase = "/api/v1" + apiNamespaces = "/namespaces" + apiServices = "/services" ) // Defaults const ( - defaultBaseUrl = "http://localhost:8080" + defaultBaseURL = "http://localhost:8080" ) - type K8sConnector struct { - baseUrl string + baseURL string } -func (c *K8sConnector) SetBaseUrl(u string) error { - validUrl, error := url.Parse(u) +func (c *K8sConnector) SetBaseURL(u string) error { + url, error := url.Parse(u) - if error != nil { - return error - } - c.baseUrl = validUrl.String() + if error != nil { + return error + } - return nil + if !url.IsAbs() { + return errors.New("k8sclient: Kubernetes endpoint url must be an absolute URL") + } + + c.baseURL = url.String() + return nil } -func (c *K8sConnector) GetBaseUrl() string { - return c.baseUrl +func (c *K8sConnector) GetBaseURL() string { + return c.baseURL } - -func (c *K8sConnector) GetResourceList() *ResourceList { - resources := new(ResourceList) - - error := getJson((c.baseUrl + apiBase), resources) - if error != nil { - return nil - } - - return resources +// URL constructor separated from code to support dependency injection +// for unit tests. +var makeURL = func(parts []string) string { + return strings.Join(parts, "") } +func (c *K8sConnector) GetResourceList() (*ResourceList, error) { + resources := new(ResourceList) -func (c *K8sConnector) GetNamespaceList() *NamespaceList { - namespaces := new(NamespaceList) + url := makeURL([]string{c.baseURL, apiBase}) + err := parseJson(url, resources) + // TODO: handle no response from k8s + if err != nil { + fmt.Printf("[ERROR] Response from kubernetes API for GetResourceList() is: %v\n", err) + return nil, err + } - error := getJson((c.baseUrl + apiBase + apiNamespaces), namespaces) - if error != nil { - return nil - } - - return namespaces + return resources, nil } +func (c *K8sConnector) GetNamespaceList() (*NamespaceList, error) { + namespaces := new(NamespaceList) -func (c *K8sConnector) GetServiceList() *ServiceList { - services := new(ServiceList) + url := makeURL([]string{c.baseURL, apiBase, apiNamespaces}) + err := parseJson(url, namespaces) + if err != nil { + fmt.Printf("[ERROR] Response from kubernetes API for GetNamespaceList() is: %v\n", err) + return nil, err + } - error := getJson((c.baseUrl + apiBase + apiServices), services) - if error != nil { - return nil - } - - return services + return namespaces, nil } +func (c *K8sConnector) GetServiceList() (*ServiceList, error) { + services := new(ServiceList) -func (c *K8sConnector) GetServicesByNamespace() map[string][]ServiceItem { - // GetServicesByNamespace returns a map of namespacename :: [ kubernetesServiceItem ] + url := makeURL([]string{c.baseURL, apiBase, apiServices}) + err := parseJson(url, services) + // TODO: handle no response from k8s + if err != nil { + fmt.Printf("[ERROR] Response from kubernetes API for GetServiceList() is: %v\n", err) + return nil, err + } - items := make(map[string][]ServiceItem) - - k8sServiceList := c.GetServiceList() - k8sItemList := k8sServiceList.Items - - for _, i := range k8sItemList { - namespace := i.Metadata.Namespace - items[namespace] = append(items[namespace], i) - } - - return items + return services, nil } +// GetServicesByNamespace returns a map of +// namespacename :: [ kubernetesServiceItem ] +func (c *K8sConnector) GetServicesByNamespace() (map[string][]ServiceItem, error) { -func (c *K8sConnector) GetServiceItemInNamespace(namespace string, servicename string) *ServiceItem { - // GetServiceItemInNamespace returns the ServiceItem that matches servicename in the namespace + items := make(map[string][]ServiceItem) - itemMap := c.GetServicesByNamespace() + k8sServiceList, err := c.GetServiceList() - // TODO: Handle case where namesapce == nil + if err != nil { + fmt.Printf("[ERROR] Getting service list produced error: %v", err) + return nil, err + } - for _, x := range itemMap[namespace] { - if x.Metadata.Name == servicename { - return &x - } - } + // TODO: handle no response from k8s + if k8sServiceList == nil { + return nil, nil + } - // No matching item found in namespace - return nil + k8sItemList := k8sServiceList.Items + + for _, i := range k8sItemList { + namespace := i.Metadata.Namespace + items[namespace] = append(items[namespace], i) + } + + return items, nil } +// GetServiceItemsInNamespace returns the ServiceItems that match +// servicename in the namespace +func (c *K8sConnector) GetServiceItemsInNamespace(namespace string, servicename string) ([]*ServiceItem, error) { -func NewK8sConnector(baseurl string) *K8sConnector { - k := new(K8sConnector) - k.SetBaseUrl(baseurl) + itemMap, err := c.GetServicesByNamespace() - return k + if err != nil { + fmt.Printf("[ERROR] Getting service list produced error: %v", err) + return nil, err + } + + // TODO: Handle case where namespace == nil + + var serviceItems []*ServiceItem + + for _, x := range itemMap[namespace] { + if x.Metadata.Name == servicename { + serviceItems = append(serviceItems, &x) + } + } + + return serviceItems, nil +} + +func NewK8sConnector(baseURL string) *K8sConnector { + k := new(K8sConnector) + + if baseURL == "" { + baseURL = defaultBaseURL + } + + err := k.SetBaseURL(baseURL) + if err != nil { + return nil + } + + return k } diff --git a/middleware/kubernetes/k8sclient/k8sclient_test.go b/middleware/kubernetes/k8sclient/k8sclient_test.go new file mode 100644 index 000000000..eded61b92 --- /dev/null +++ b/middleware/kubernetes/k8sclient/k8sclient_test.go @@ -0,0 +1,680 @@ +package k8sclient + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +var validURLs = []string{ + "http://www.github.com", + "http://www.github.com:8080", + "http://8.8.8.8", + "http://8.8.8.8:9090", + "www.github.com:8080", +} + +var invalidURLs = []string{ + "www.github.com", + "8.8.8.8", + "8.8.8.8:1010", + "8.8`8.8", +} + +func TestNewK8sConnector(t *testing.T) { + var conn *K8sConnector + var url string + + // Create with empty URL + conn = nil + url = "" + + conn = NewK8sConnector("") + if conn == nil { + t.Errorf("Expected K8sConnector instance. Instead got '%v'", conn) + } + url = conn.GetBaseURL() + if url != defaultBaseURL { + t.Errorf("Expected K8sConnector instance to be initialized with defaultBaseURL. Instead got '%v'", url) + } + + // Create with valid URL + for _, validURL := range validURLs { + conn = nil + url = "" + + conn = NewK8sConnector(validURL) + if conn == nil { + t.Errorf("Expected K8sConnector instance. Instead got '%v'", conn) + } + url = conn.GetBaseURL() + if url != validURL { + t.Errorf("Expected K8sConnector instance to be initialized with supplied url '%v'. Instead got '%v'", validURL, url) + } + } + + // Create with invalid URL + for _, invalidURL := range invalidURLs { + conn = nil + url = "" + + conn = NewK8sConnector(invalidURL) + if conn != nil { + t.Errorf("Expected to not get K8sConnector instance. Instead got '%v'", conn) + continue + } + } +} + +func TestSetBaseURL(t *testing.T) { + // SetBaseURL with valid URLs should work... + for _, validURL := range validURLs { + conn := NewK8sConnector(defaultBaseURL) + err := conn.SetBaseURL(validURL) + if err != nil { + t.Errorf("Expected to receive nil, instead got error '%v'", err) + continue + } + url := conn.GetBaseURL() + if url != validURL { + t.Errorf("Expected to connector url to be set to value '%v', instead set to '%v'", validURL, url) + continue + } + } + + // SetBaseURL with invalid or non absolute URLs should not change state... + for _, invalidURL := range invalidURLs { + conn := NewK8sConnector(defaultBaseURL) + originalURL := conn.GetBaseURL() + + err := conn.SetBaseURL(invalidURL) + if err == nil { + t.Errorf("Expected to receive an error value, instead got nil") + } + url := conn.GetBaseURL() + if url != originalURL { + t.Errorf("Expected base url to not change, instead it changed to '%v'", url) + } + } +} + +func TestGetNamespaceList(t *testing.T) { + // Set up a test http server + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, namespaceListJsonData) + })) + defer testServer.Close() + + // Overwrite URL constructor to access testServer + makeURL = func(parts []string) string { + return testServer.URL + } + + expectedNamespaces := []string{"default", "demo", "test"} + apiConn := NewK8sConnector("") + namespaceList, err := apiConn.GetNamespaceList() + + if err != nil { + t.Errorf("Expected no error from from GetNamespaceList(), instead got %v", err) + } + + if namespaceList == nil { + t.Errorf("Expected data from GetNamespaceList(), instead got nil") + } + + kind := namespaceList.Kind + if kind != "NamespaceList" { + t.Errorf("Expected data from GetNamespaceList() to have Kind='NamespaceList', instead got Kind='%v'", kind) + } + + // Ensure correct number of namespaces found + expectedCount := len(expectedNamespaces) + namespaceCount := len(namespaceList.Items) + if namespaceCount != expectedCount { + t.Errorf("Expected '%v' namespaces from GetNamespaceList(), instead found '%v' namespaces", expectedCount, namespaceCount) + } + + // Check that all expectedNamespaces are found in the parsed data + for _, ns := range expectedNamespaces { + found := false + for _, item := range namespaceList.Items { + if item.Metadata.Name == ns { + found = true + break + } + } + if !found { + t.Errorf("Expected '%v' namespace is not in the parsed data from GetServicesByNamespace()", ns) + } + } +} + +func TestGetServiceList(t *testing.T) { + // Set up a test http server + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, serviceListJsonData) + })) + defer testServer.Close() + + // Overwrite URL constructor to access testServer + makeURL = func(parts []string) string { + return testServer.URL + } + + expectedServices := []string{"kubernetes", "mynginx", "mywebserver"} + apiConn := NewK8sConnector("") + serviceList, err := apiConn.GetServiceList() + + if err != nil { + t.Errorf("Expected no error from from GetNamespaceList(), instead got %v", err) + } + + if serviceList == nil { + t.Errorf("Expected data from GetServiceList(), instead got nil") + } + + kind := serviceList.Kind + if kind != "ServiceList" { + t.Errorf("Expected data from GetServiceList() to have Kind='ServiceList', instead got Kind='%v'", kind) + } + + // Ensure correct number of services found + expectedCount := len(expectedServices) + serviceCount := len(serviceList.Items) + if serviceCount != expectedCount { + t.Errorf("Expected '%v' services from GetServiceList(), instead found '%v' services", expectedCount, serviceCount) + } + + // Check that all expectedServices are found in the parsed data + for _, s := range expectedServices { + found := false + for _, item := range serviceList.Items { + if item.Metadata.Name == s { + found = true + break + } + } + if !found { + t.Errorf("Expected '%v' service is not in the parsed data from GetServiceList()", s) + } + } +} + +func TestGetServicesByNamespace(t *testing.T) { + // Set up a test http server + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, serviceListJsonData) + })) + defer testServer.Close() + + // Overwrite URL constructor to access testServer + makeURL = func(parts []string) string { + return testServer.URL + } + + expectedNamespaces := []string{"default", "demo"} + apiConn := NewK8sConnector("") + servicesByNamespace, err := apiConn.GetServicesByNamespace() + + if err != nil { + t.Errorf("Expected no error from from GetServicesByNamespace(), instead got %v", err) + } + + // Ensure correct number of namespaces found + expectedCount := len(expectedNamespaces) + namespaceCount := len(servicesByNamespace) + if namespaceCount != expectedCount { + t.Errorf("Expected '%v' namespaces from GetServicesByNamespace(), instead found '%v' namespaces", expectedCount, namespaceCount) + } + + // Check that all expectedNamespaces are found in the parsed data + for _, ns := range expectedNamespaces { + _, ok := servicesByNamespace[ns] + if !ok { + t.Errorf("Expected '%v' namespace is not in the parsed data from GetServicesByNamespace()", ns) + } + } +} + +func TestGetResourceList(t *testing.T) { + // Set up a test http server + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, resourceListJsonData) + })) + defer testServer.Close() + + // Overwrite URL constructor to access testServer + makeURL = func(parts []string) string { + return testServer.URL + } + + expectedResources := []string{"bindings", + "componentstatuses", + "configmaps", + "endpoints", + "events", + "limitranges", + "namespaces", + "namespaces/finalize", + "namespaces/status", + "nodes", + "nodes/proxy", + "nodes/status", + "persistentvolumeclaims", + "persistentvolumeclaims/status", + "persistentvolumes", + "persistentvolumes/status", + "pods", + "pods/attach", + "pods/binding", + "pods/exec", + "pods/log", + "pods/portforward", + "pods/proxy", + "pods/status", + "podtemplates", + "replicationcontrollers", + "replicationcontrollers/scale", + "replicationcontrollers/status", + "resourcequotas", + "resourcequotas/status", + "secrets", + "serviceaccounts", + "services", + "services/proxy", + "services/status", + } + apiConn := NewK8sConnector("") + resourceList, err := apiConn.GetResourceList() + + if err != nil { + t.Errorf("Expected no error from from GetResourceList(), instead got %v", err) + } + + if resourceList == nil { + t.Errorf("Expected data from GetResourceList(), instead got nil") + } + + kind := resourceList.Kind + if kind != "APIResourceList" { + t.Errorf("Expected data from GetResourceList() to have Kind='ResourceList', instead got Kind='%v'", kind) + } + + // Ensure correct number of resources found + expectedCount := len(expectedResources) + resourceCount := len(resourceList.Resources) + if resourceCount != expectedCount { + t.Errorf("Expected '%v' resources from GetResourceList(), instead found '%v' resources", expectedCount, resourceCount) + } + + // Check that all expectedResources are found in the parsed data + for _, r := range expectedResources { + found := false + for _, item := range resourceList.Resources { + if item.Name == r { + found = true + break + } + } + if !found { + t.Errorf("Expected '%v' resource is not in the parsed data from GetResourceList()", r) + } + } +} + +// Sample namespace data for kubernetes with 3 namespaces: +// "default", "demo", and "test". +const namespaceListJsonData string = `{ + "kind": "NamespaceList", + "apiVersion": "v1", + "metadata": { + "selfLink": "/api/v1/namespaces/", + "resourceVersion": "121279" + }, + "items": [ + { + "metadata": { + "name": "default", + "selfLink": "/api/v1/namespaces/default", + "uid": "fb1c92d1-2f39-11e6-b9db-0800279930f6", + "resourceVersion": "6", + "creationTimestamp": "2016-06-10T18:34:35Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + }, + { + "metadata": { + "name": "demo", + "selfLink": "/api/v1/namespaces/demo", + "uid": "73be8ffd-2f3a-11e6-b9db-0800279930f6", + "resourceVersion": "111", + "creationTimestamp": "2016-06-10T18:37:57Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + }, + { + "metadata": { + "name": "test", + "selfLink": "/api/v1/namespaces/test", + "uid": "c0be05fa-3352-11e6-b9db-0800279930f6", + "resourceVersion": "121276", + "creationTimestamp": "2016-06-15T23:41:59Z" + }, + "spec": { + "finalizers": [ + "kubernetes" + ] + }, + "status": { + "phase": "Active" + } + } + ] +}` + +// Sample service data for kubernetes with 3 services: +// * "kubernetes" (in "default" namespace) +// * "mynginx" (in "demo" namespace) +// * "webserver" (in "demo" namespace) +const serviceListJsonData string = ` +{ + "kind": "ServiceList", + "apiVersion": "v1", + "metadata": { + "selfLink": "/api/v1/services", + "resourceVersion": "147965" + }, + "items": [ + { + "metadata": { + "name": "kubernetes", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/services/kubernetes", + "uid": "fb1cb0d3-2f39-11e6-b9db-0800279930f6", + "resourceVersion": "7", + "creationTimestamp": "2016-06-10T18:34:35Z", + "labels": { + "component": "apiserver", + "provider": "kubernetes" + } + }, + "spec": { + "ports": [ + { + "name": "https", + "protocol": "TCP", + "port": 443, + "targetPort": 443 + } + ], + "clusterIP": "10.0.0.1", + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + }, + { + "metadata": { + "name": "mynginx", + "namespace": "demo", + "selfLink": "/api/v1/namespaces/demo/services/mynginx", + "uid": "93c117ac-2f3a-11e6-b9db-0800279930f6", + "resourceVersion": "147", + "creationTimestamp": "2016-06-10T18:38:51Z", + "labels": { + "run": "mynginx" + } + }, + "spec": { + "ports": [ + { + "protocol": "TCP", + "port": 80, + "targetPort": 80 + } + ], + "selector": { + "run": "mynginx" + }, + "clusterIP": "10.0.0.132", + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + }, + { + "metadata": { + "name": "mywebserver", + "namespace": "demo", + "selfLink": "/api/v1/namespaces/demo/services/mywebserver", + "uid": "aed62187-33e5-11e6-a224-0800279930f6", + "resourceVersion": "138185", + "creationTimestamp": "2016-06-16T17:13:45Z", + "labels": { + "run": "mywebserver" + } + }, + "spec": { + "ports": [ + { + "protocol": "TCP", + "port": 443, + "targetPort": 443 + } + ], + "selector": { + "run": "mywebserver" + }, + "clusterIP": "10.0.0.63", + "type": "ClusterIP", + "sessionAffinity": "None" + }, + "status": { + "loadBalancer": {} + } + } + ] +} +` + +// Sample resource data for kubernetes. +const resourceListJsonData string = `{ + "kind": "APIResourceList", + "groupVersion": "v1", + "resources": [ + { + "name": "bindings", + "namespaced": true, + "kind": "Binding" + }, + { + "name": "componentstatuses", + "namespaced": false, + "kind": "ComponentStatus" + }, + { + "name": "configmaps", + "namespaced": true, + "kind": "ConfigMap" + }, + { + "name": "endpoints", + "namespaced": true, + "kind": "Endpoints" + }, + { + "name": "events", + "namespaced": true, + "kind": "Event" + }, + { + "name": "limitranges", + "namespaced": true, + "kind": "LimitRange" + }, + { + "name": "namespaces", + "namespaced": false, + "kind": "Namespace" + }, + { + "name": "namespaces/finalize", + "namespaced": false, + "kind": "Namespace" + }, + { + "name": "namespaces/status", + "namespaced": false, + "kind": "Namespace" + }, + { + "name": "nodes", + "namespaced": false, + "kind": "Node" + }, + { + "name": "nodes/proxy", + "namespaced": false, + "kind": "Node" + }, + { + "name": "nodes/status", + "namespaced": false, + "kind": "Node" + }, + { + "name": "persistentvolumeclaims", + "namespaced": true, + "kind": "PersistentVolumeClaim" + }, + { + "name": "persistentvolumeclaims/status", + "namespaced": true, + "kind": "PersistentVolumeClaim" + }, + { + "name": "persistentvolumes", + "namespaced": false, + "kind": "PersistentVolume" + }, + { + "name": "persistentvolumes/status", + "namespaced": false, + "kind": "PersistentVolume" + }, + { + "name": "pods", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/attach", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/binding", + "namespaced": true, + "kind": "Binding" + }, + { + "name": "pods/exec", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/log", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/portforward", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/proxy", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "pods/status", + "namespaced": true, + "kind": "Pod" + }, + { + "name": "podtemplates", + "namespaced": true, + "kind": "PodTemplate" + }, + { + "name": "replicationcontrollers", + "namespaced": true, + "kind": "ReplicationController" + }, + { + "name": "replicationcontrollers/scale", + "namespaced": true, + "kind": "Scale" + }, + { + "name": "replicationcontrollers/status", + "namespaced": true, + "kind": "ReplicationController" + }, + { + "name": "resourcequotas", + "namespaced": true, + "kind": "ResourceQuota" + }, + { + "name": "resourcequotas/status", + "namespaced": true, + "kind": "ResourceQuota" + }, + { + "name": "secrets", + "namespaced": true, + "kind": "Secret" + }, + { + "name": "serviceaccounts", + "namespaced": true, + "kind": "ServiceAccount" + }, + { + "name": "services", + "namespaced": true, + "kind": "Service" + }, + { + "name": "services/proxy", + "namespaced": true, + "kind": "Service" + }, + { + "name": "services/status", + "namespaced": true, + "kind": "Service" + } + ] +}` diff --git a/middleware/kubernetes/kubernetes.go b/middleware/kubernetes/kubernetes.go index 25c8cab3c..d6d93f809 100644 --- a/middleware/kubernetes/kubernetes.go +++ b/middleware/kubernetes/kubernetes.go @@ -2,111 +2,148 @@ package kubernetes import ( - "fmt" - "strings" + "errors" + "fmt" "time" "github.com/miekg/coredns/middleware" - "github.com/miekg/coredns/middleware/kubernetes/msg" k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" + "github.com/miekg/coredns/middleware/kubernetes/msg" + "github.com/miekg/coredns/middleware/kubernetes/nametemplate" + "github.com/miekg/coredns/middleware/kubernetes/util" "github.com/miekg/coredns/middleware/proxy" -// "github.com/miekg/coredns/middleware/singleflight" + // "github.com/miekg/coredns/middleware/singleflight" - "github.com/miekg/dns" + "github.com/miekg/dns" "golang.org/x/net/context" ) type Kubernetes struct { - Next middleware.Handler - Zones []string - Proxy proxy.Proxy // Proxy for looking up names during the resolution process - Ctx context.Context -// Inflight *singleflight.Group - APIConn *k8sc.K8sConnector + Next middleware.Handler + Zones []string + Proxy proxy.Proxy // Proxy for looking up names during the resolution process + Ctx context.Context + // Inflight *singleflight.Group + APIConn *k8sc.K8sConnector + NameTemplate *nametemplate.NameTemplate + Namespaces *[]string } - +// getZoneForName returns the zone string that matches the name and a +// list of the DNS labels from name that are within the zone. +// For example, if "coredns.local" is a zone configured for the +// Kubernetes middleware, then getZoneForName("a.b.coredns.local") +// will return ("coredns.local", ["a", "b"]). func (g Kubernetes) getZoneForName(name string) (string, []string) { - /* - * getZoneForName returns the zone string that matches the name and a - * list of the DNS labels from name that are within the zone. - * For example, if "coredns.local" is a zone configured for the - * Kubernetes middleware, then getZoneForName("a.b.coredns.local") - * will return ("coredns.local", ["a", "b"]). - */ - var zone string - var serviceSegments []string + var zone string + var serviceSegments []string - for _, z := range g.Zones { - if dns.IsSubDomain(z, name) { - zone = z - - serviceSegments = dns.SplitDomainName(name) - serviceSegments = serviceSegments[:len(serviceSegments) - dns.CountLabel(zone)] - break - } - } + for _, z := range g.Zones { + if dns.IsSubDomain(z, name) { + zone = z - return zone, serviceSegments -} + serviceSegments = dns.SplitDomainName(name) + serviceSegments = serviceSegments[:len(serviceSegments)-dns.CountLabel(zone)] + break + } + } + return zone, serviceSegments +} // Records looks up services in kubernetes. // If exact is true, it will lookup just // this name. This is used when find matches when completing SRV lookups // for instance. func (g Kubernetes) Records(name string, exact bool) ([]msg.Service, error) { + var ( + serviceName string + namespace string + typeName string + ) - fmt.Println("enter Records('", name, "', ", exact, ")") + fmt.Println("enter Records('", name, "', ", exact, ")") + zone, serviceSegments := g.getZoneForName(name) - zone, serviceSegments := g.getZoneForName(name) + /* + // For initial implementation, assume namespace is first serviceSegment + // and service name is remaining segments. + serviceSegLen := len(serviceSegments) + if serviceSegLen >= 2 { + namespace = serviceSegments[serviceSegLen-1] + serviceName = strings.Join(serviceSegments[:serviceSegLen-1], ".") + } + // else we are looking up the zone. So handle the NS, SOA records etc. + */ - var serviceName string - var namespace string + // TODO: Implementation above globbed together segments for the serviceName if + // multiple segments remained. Determine how to do similar globbing using + // the template-based implementation. + namespace = g.NameTemplate.GetNamespaceFromSegmentArray(serviceSegments) + serviceName = g.NameTemplate.GetServiceFromSegmentArray(serviceSegments) + typeName = g.NameTemplate.GetTypeFromSegmentArray(serviceSegments) - // For initial implementation, assume namespace is first serviceSegment - // and service name is remaining segments. - serviceSegLen := len(serviceSegments) - if serviceSegLen >= 2 { - namespace = serviceSegments[serviceSegLen-1] - serviceName = strings.Join(serviceSegments[:serviceSegLen-1], ".") - } - // else we are looking up the zone. So handle the NS, SOA records etc. + fmt.Println("[debug] exact: ", exact) + fmt.Println("[debug] zone: ", zone) + fmt.Println("[debug] servicename: ", serviceName) + fmt.Println("[debug] namespace: ", namespace) + fmt.Println("[debug] typeName: ", typeName) + fmt.Println("[debug] APIconn: ", g.APIConn) - fmt.Println("[debug] zone: ", zone) - fmt.Println("[debug] servicename: ", serviceName) - fmt.Println("[debug] namespace: ", namespace) - fmt.Println("[debug] APIconn: ", g.APIConn) + // TODO: Implement wildcard support to allow blank namespace value + if namespace == "" { + err := errors.New("Parsing query string did not produce a namespace value") + fmt.Printf("[ERROR] %v\n", err) + return nil, err + } - k8sItem := g.APIConn.GetServiceItemInNamespace(namespace, serviceName) - fmt.Println("[debug] k8s item:", k8sItem) + // Abort if the namespace is not published per CoreFile + if g.Namespaces != nil && !util.StringInSlice(namespace, *g.Namespaces) { + return nil, nil + } - switch { - case exact && k8sItem == nil: - fmt.Println("here2") - return nil, nil - } + k8sItems, err := g.APIConn.GetServiceItemsInNamespace(namespace, serviceName) + fmt.Println("[debug] k8s items:", k8sItems) - if k8sItem == nil { - // Did not find item in k8s - return nil, nil - } + if err != nil { + fmt.Printf("[ERROR] Got error while looking up ServiceItems. Error is: %v\n", err) + return nil, err + } + if k8sItems == nil { + // Did not find item in k8s + return nil, nil + } - fmt.Println("[debug] clusterIP:", k8sItem.Spec.ClusterIP) + // test := g.NameTemplate.GetRecordNameFromNameValues(nametemplate.NameValues{ServiceName: serviceName, TypeName: typeName, Namespace: namespace, Zone: zone}) + // fmt.Printf("[debug] got recordname %v\n", test) - for _, p := range k8sItem.Spec.Ports { - fmt.Println("[debug] host:", name) - fmt.Println("[debug] port:", p.Port) - } + records := g.getRecordsForServiceItems(k8sItems, name) - clusterIP := k8sItem.Spec.ClusterIP - var records []msg.Service - for _, p := range k8sItem.Spec.Ports{ - s := msg.Service{Host: clusterIP, Port: p.Port} - records = append(records, s) - } + return records, nil +} - return records, nil +// TODO: assemble name from parts found in k8s data based on name template rather than reusing query string +func (g Kubernetes) getRecordsForServiceItems(serviceItems []*k8sc.ServiceItem, name string) []msg.Service { + var records []msg.Service + + for _, item := range serviceItems { + fmt.Println("[debug] clusterIP:", item.Spec.ClusterIP) + for _, p := range item.Spec.Ports { + fmt.Println("[debug] port:", p.Port) + } + + clusterIP := item.Spec.ClusterIP + + s := msg.Service{Host: name} + records = append(records, s) + for _, p := range item.Spec.Ports { + s := msg.Service{Host: clusterIP, Port: p.Port} + records = append(records, s) + } + } + + fmt.Printf("[debug] records from getRecordsForServiceItems(): %v\n", records) + return records } /* @@ -121,13 +158,13 @@ func (g Kubernetes) Get(path string, recursive bool) (bool, error) { */ func (g Kubernetes) splitDNSName(name string) []string { - l := dns.SplitDomainName(name) + l := dns.SplitDomainName(name) - for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { - l[i], l[j] = l[j], l[i] - } + for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] + } - return l + return l } // skydns/local/skydns/east/staging/web @@ -215,9 +252,9 @@ func isKubernetesNameError(err error) bool { } const ( - priority = 10 // default priority when nothing is set - ttl = 300 // default ttl when nothing is set - minTtl = 60 - hostmaster = "hostmaster" + priority = 10 // default priority when nothing is set + ttl = 300 // default ttl when nothing is set + minTtl = 60 + hostmaster = "hostmaster" k8sTimeout = 5 * time.Second ) diff --git a/middleware/kubernetes/lookup.go b/middleware/kubernetes/lookup.go index 1efec7475..b490d6a4b 100644 --- a/middleware/kubernetes/lookup.go +++ b/middleware/kubernetes/lookup.go @@ -17,7 +17,7 @@ func (k Kubernetes) records(state middleware.State, exact bool) ([]msg.Service, if err != nil { return nil, err } - // TODO: Do we want to support the SkyDNS (hacky) Group feature? + // TODO: Do we want to support the SkyDNS (hacky) Group feature? services = msg.Group(services) return services, nil } @@ -141,7 +141,7 @@ func (k Kubernetes) AAAA(zone string, state middleware.State, previousRecords [] return records, nil } -// SRV returns SRV records from etcd. +// SRV returns SRV records from kubernetes. // If the Target is not a name but an IP address, a name is created on the fly. func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { services, err := k.records(state, false) @@ -208,13 +208,13 @@ func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, } // k.AAA(zone, state1, nil) as well...? case ip.To4() != nil: - serv.Host = k.Domain(serv.Key) + serv.Host = serv.Key srv := serv.NewSRV(state.QName(), weight) records = append(records, srv) extra = append(extra, serv.NewA(srv.Target, ip.To4())) case ip.To4() == nil: - serv.Host = k.Domain(serv.Key) + serv.Host = serv.Key srv := serv.NewSRV(state.QName(), weight) records = append(records, srv) @@ -226,17 +226,17 @@ func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, // Returning MX records from kubernetes not implemented. func (k Kubernetes) MX(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { - return nil, nil, err + return nil, nil, err } // Returning CNAME records from kubernetes not implemented. func (k Kubernetes) CNAME(zone string, state middleware.State) (records []dns.RR, err error) { - return nil, err + return nil, err } // Returning TXT records from kubernetes not implemented. func (k Kubernetes) TXT(zone string, state middleware.State) (records []dns.RR, err error) { - return nil, err + return nil, err } func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dns.RR, err error) { @@ -259,11 +259,11 @@ func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dn case ip == nil: return nil, nil, fmt.Errorf("NS record must be an IP address: %s", serv.Host) case ip.To4() != nil: - serv.Host = k.Domain(serv.Key) + serv.Host = serv.Key records = append(records, serv.NewNS(state.QName())) extra = append(extra, serv.NewA(serv.Host, ip.To4())) case ip.To4() == nil: - serv.Host = k.Domain(serv.Key) + serv.Host = serv.Key records = append(records, serv.NewNS(state.QName())) extra = append(extra, serv.NewAAAA(serv.Host, ip.To16())) } diff --git a/middleware/kubernetes/nametemplate/nametemplate.go b/middleware/kubernetes/nametemplate/nametemplate.go new file mode 100644 index 000000000..e61c036e8 --- /dev/null +++ b/middleware/kubernetes/nametemplate/nametemplate.go @@ -0,0 +1,166 @@ +package nametemplate + +import ( + "errors" + "fmt" + "strings" + + "github.com/miekg/coredns/middleware/kubernetes/util" +) + +// Likely symbols that require support: +// {id} +// {ip} +// {portname} +// {protocolname} +// {servicename} +// {namespace} +// {type} "svc" or "pod" +// {zone} + +// SkyDNS normal services have an A-record of the form "{servicename}.{namespace}.{type}.{zone}" +// This resolves to the cluster IP of the service. + +// SkyDNS headless services have an A-record of the form "{servicename}.{namespace}.{type}.{zone}" +// This resolves to the set of IPs of the pods selected by the Service. Clients are expected to +// consume the set or else use round-robin selection from the set. + +var symbols = map[string]string{ + "service": "{service}", + "namespace": "{namespace}", + "type": "{type}", + "zone": "{zone}", +} + +var types = []string{ + "svc", + "pod", +} + +// TODO: Validate that provided NameTemplate string only contains: +// * valid, known symbols, or +// * static strings + +// TODO: Support collapsing multiple segments into a symbol. Either: +// * all left-over segments are used as the "service" name, or +// * some scheme like "{namespace}.{namespace}" means use +// segments concatenated with a "." for the namespace, or +// * {namespace2:4} means use segements 2->4 for the namespace. + +// TODO: possibly need to store length of segmented format to handle cases +// where query string segments to a shorter or longer list than the template. +// When query string segments to shorter than template: +// * either wildcards are being used, or +// * we are not looking up an A, AAAA, or SRV record (eg NS), or +// * we can just short-circuit failure before hitting the k8s API. +// Where the query string is longer than the template, need to define which +// symbol consumes the other segments. Most likely this would be the servicename. +// Also consider how to handle static strings in the format template. +type NameTemplate struct { + formatString string + splitFormat []string + // Element is a map of element name :: index in the segmented record name for the named element + Element map[string]int +} + +func (t *NameTemplate) SetTemplate(s string) error { + var err error + fmt.Println() + + t.Element = map[string]int{} + + t.formatString = s + t.splitFormat = strings.Split(t.formatString, ".") + for templateIndex, v := range t.splitFormat { + elementPositionSet := false + for name, symbol := range symbols { + if v == symbol { + t.Element[name] = templateIndex + elementPositionSet = true + break + } + } + if !elementPositionSet { + if strings.Contains(v, "{") { + err = errors.New("Record name template contains the unknown symbol '" + v + "'") + fmt.Printf("[debug] %v\n", err) + return err + } else { + fmt.Printf("[debug] Template string has static element '%v'\n", v) + } + } + } + + return err +} + +// TODO: Find a better way to pull the data segments out of the +// query string based on the template. Perhaps it is better +// to treat the query string segments as a reverse stack and +// step down the stack to find the right element. + +func (t *NameTemplate) GetZoneFromSegmentArray(segments []string) string { + if index, ok := t.Element["zone"]; !ok { + return "" + } else { + return strings.Join(segments[index:len(segments)], ".") + } +} + +func (t *NameTemplate) GetNamespaceFromSegmentArray(segments []string) string { + return t.GetSymbolFromSegmentArray("namespace", segments) +} + +func (t *NameTemplate) GetServiceFromSegmentArray(segments []string) string { + return t.GetSymbolFromSegmentArray("service", segments) +} + +func (t *NameTemplate) GetTypeFromSegmentArray(segments []string) string { + typeSegment := t.GetSymbolFromSegmentArray("type", segments) + + // Limit type to known types symbols + if util.StringInSlice(typeSegment, types) { + return "" + } + + return typeSegment +} + +func (t *NameTemplate) GetSymbolFromSegmentArray(symbol string, segments []string) string { + if index, ok := t.Element[symbol]; !ok { + return "" + } else { + return segments[index] + } +} + +// GetRecordNameFromNameValues returns the string produced by applying the +// values to the NameTemplate format string. +func (t *NameTemplate) GetRecordNameFromNameValues(values NameValues) string { + recordName := make([]string, len(t.splitFormat)) + copy(recordName[:], t.splitFormat) + + for name, index := range t.Element { + if index == -1 { + continue + } + switch name { + case "type": + recordName[index] = values.TypeName + case "service": + recordName[index] = values.ServiceName + case "namespace": + recordName[index] = values.Namespace + case "zone": + recordName[index] = values.Zone + } + } + return strings.Join(recordName, ".") +} + +type NameValues struct { + ServiceName string + Namespace string + TypeName string + Zone string +} diff --git a/middleware/kubernetes/nametemplate/nametemplate_test.go b/middleware/kubernetes/nametemplate/nametemplate_test.go new file mode 100644 index 000000000..a866c4fb6 --- /dev/null +++ b/middleware/kubernetes/nametemplate/nametemplate_test.go @@ -0,0 +1,129 @@ +package nametemplate + +import ( + "fmt" + "strings" + "testing" +) + +const ( + zone = 0 + namespace = 1 + service = 2 +) + +// Map of format string :: expected locations of name symbols in the format. +// -1 value indicates that symbol does not exist in format. +var exampleTemplates = map[string][]int{ + "{service}.{namespace}.{zone}": []int{2, 1, 0}, // service symbol expected @ position 0, namespace @ 1, zone @ 2 + "{namespace}.{zone}": []int{1, 0, -1}, + "": []int{-1, -1, -1}, +} + +func TestSetTemplate(t *testing.T) { + fmt.Printf("\n") + for s, expectedValue := range exampleTemplates { + + n := new(NameTemplate) + n.SetTemplate(s) + + // check the indexes resulting from calling SetTemplate() against expectedValues + if expectedValue[zone] != -1 { + if n.Element["zone"] != expectedValue[zone] { + t.Errorf("Expected zone at index '%v', instead found at index '%v' for format string '%v'", expectedValue[zone], n.Element["zone"], s) + } + } + } +} + +func TestGetServiceFromSegmentArray(t *testing.T) { + var ( + n *NameTemplate + formatString string + queryString string + splitQuery []string + expectedService string + actualService string + ) + + // Case where template contains {service} + n = new(NameTemplate) + formatString = "{service}.{namespace}.{zone}" + n.SetTemplate(formatString) + + queryString = "myservice.mynamespace.coredns" + splitQuery = strings.Split(queryString, ".") + expectedService = "myservice" + actualService = n.GetServiceFromSegmentArray(splitQuery) + + if actualService != expectedService { + t.Errorf("Expected service name '%v', instead got service name '%v' for query string '%v' and format '%v'", expectedService, actualService, queryString, formatString) + } + + // Case where template does not contain {service} + n = new(NameTemplate) + formatString = "{namespace}.{zone}" + n.SetTemplate(formatString) + + queryString = "mynamespace.coredns" + splitQuery = strings.Split(queryString, ".") + expectedService = "" + actualService = n.GetServiceFromSegmentArray(splitQuery) + + if actualService != expectedService { + t.Errorf("Expected service name '%v', instead got service name '%v' for query string '%v' and format '%v'", expectedService, actualService, queryString, formatString) + } +} + +func TestGetZoneFromSegmentArray(t *testing.T) { + var ( + n *NameTemplate + formatString string + queryString string + splitQuery []string + expectedZone string + actualZone string + ) + + // Case where template contains {zone} + n = new(NameTemplate) + formatString = "{service}.{namespace}.{zone}" + n.SetTemplate(formatString) + + queryString = "myservice.mynamespace.coredns" + splitQuery = strings.Split(queryString, ".") + expectedZone = "coredns" + actualZone = n.GetZoneFromSegmentArray(splitQuery) + + if actualZone != expectedZone { + t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString) + } + + // Case where template does not contain {zone} + n = new(NameTemplate) + formatString = "{service}.{namespace}" + n.SetTemplate(formatString) + + queryString = "mynamespace.coredns" + splitQuery = strings.Split(queryString, ".") + expectedZone = "" + actualZone = n.GetZoneFromSegmentArray(splitQuery) + + if actualZone != expectedZone { + t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString) + } + + // Case where zone is multiple segments + n = new(NameTemplate) + formatString = "{service}.{namespace}.{zone}" + n.SetTemplate(formatString) + + queryString = "myservice.mynamespace.coredns.cluster.local" + splitQuery = strings.Split(queryString, ".") + expectedZone = "coredns.cluster.local" + actualZone = n.GetZoneFromSegmentArray(splitQuery) + + if actualZone != expectedZone { + t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString) + } +} diff --git a/middleware/kubernetes/path.go b/middleware/kubernetes/path.go deleted file mode 100644 index 18c26f949..000000000 --- a/middleware/kubernetes/path.go +++ /dev/null @@ -1,17 +0,0 @@ -package kubernetes - -import ( - "strings" - - "github.com/miekg/dns" -) - -// Domain is the opposite of Path. -func (k Kubernetes) Domain(s string) string { - l := strings.Split(s, "/") - // start with 1, to strip /skydns - for i, j := 1, len(l)-1; i < j; i, j = i+1, j-1 { - l[i], l[j] = l[j], l[i] - } - return dns.Fqdn(strings.Join(l[1:len(l)-1], ".")) -} diff --git a/middleware/kubernetes/subzone.go b/middleware/kubernetes/subzone.go new file mode 100644 index 000000000..48efba472 --- /dev/null +++ b/middleware/kubernetes/subzone.go @@ -0,0 +1,48 @@ +package kubernetes + +import ( + "fmt" + + "github.com/miekg/dns" +) + +// NormalizeZoneList filters the zones argument to remove +// array items that conflict with other items in zones. +// For example, providing the following zones array: +// [ "a.b.c", "b.c", "a", "e.d.f", "a.b" ] +// Returns: +// [ "a.b.c", "a", "e.d.f", "a.b" ] +// Zones filted out: +// - "b.c" because "a.b.c" and "b.c" share the common top +// level "b.c". First listed zone wins if there is a conflict. +// +// Note: This may prove to be too restrictive in practice. +// Need to find counter-example use-cases. +func NormalizeZoneList(zones []string) []string { + filteredZones := []string{} + + for _, z := range zones { + zoneConflict, _ := subzoneConflict(filteredZones, z) + if zoneConflict { + fmt.Printf("[WARN] new zone '%v' from Corefile conflicts with existing zones: %v\n Ignoring zone '%v'\n", z, filteredZones, z) + } else { + filteredZones = append(filteredZones, z) + } + } + + return filteredZones +} + +// subzoneConflict returns true if name is a child or parent zone of +// any element in zones. If conflicts exist, return the conflicting zones. +func subzoneConflict(zones []string, name string) (bool, []string) { + conflicts := []string{} + + for _, z := range zones { + if dns.IsSubDomain(z, name) || dns.IsSubDomain(name, z) { + conflicts = append(conflicts, z) + } + } + + return (len(conflicts) != 0), conflicts +} diff --git a/middleware/kubernetes/subzone_test.go b/middleware/kubernetes/subzone_test.go new file mode 100644 index 000000000..c48e66c12 --- /dev/null +++ b/middleware/kubernetes/subzone_test.go @@ -0,0 +1,32 @@ +package kubernetes + +import ( + "testing" +) + +// List of configured zones to test against +var confZones = []string{ + "a.b.c", + "d", +} + +// Map of zonename :: expected boolean result +var examplesSubzoneConflict = map[string]bool{ + "a.b.c": true, // conflicts with zone "a.b.c" + "b.c": true, // conflicts with zone "a.b.c" + "c": true, // conflicts with zone "a.b.c" + "e": false, // no conflict + "a.b.c.e": false, // no conflict + "a.b.c.d": true, // conflicts with zone "d" + "": false, +} + +func TestsubzoneConflict(t *testing.T) { + for z, expected := range examplesSubzoneConflict { + actual, conflicts := subzoneConflict(confZones, z) + + if actual != expected { + t.Errorf("Expected conflict result '%v' for example '%v'. Instead got '%v'. Conflicting zones are: %v", expected, z, actual, conflicts) + } + } +} diff --git a/middleware/kubernetes/util/util.go b/middleware/kubernetes/util/util.go new file mode 100644 index 000000000..7fc03ffc1 --- /dev/null +++ b/middleware/kubernetes/util/util.go @@ -0,0 +1,12 @@ +// Package kubernetes/util provides helper functions for the kubernetes middleware +package util + +// StringInSlice check whether string a is a member of slice. +func StringInSlice(a string, slice []string) bool { + for _, b := range slice { + if b == a { + return true + } + } + return false +} diff --git a/middleware/kubernetes/util/util_test.go b/middleware/kubernetes/util/util_test.go new file mode 100644 index 000000000..b53b9f3f6 --- /dev/null +++ b/middleware/kubernetes/util/util_test.go @@ -0,0 +1,33 @@ +package util + +import ( + "testing" +) + +type InSliceData struct { + Slice []string + String string + InSlice bool +} + +// Test data for TestStringInSlice cases. +var testdataInSlice = []struct { + Slice []string + String string + ExpectedResult bool +}{ + {[]string{"a", "b", "c"}, "a", true}, + {[]string{"a", "b", "c"}, "d", false}, + {[]string{"a", "b", "c"}, "", false}, + {[]string{}, "a", false}, + {[]string{}, "", false}, +} + +func TestStringInSlice(t *testing.T) { + for _, example := range testdataInSlice { + actualResult := StringInSlice(example.String, example.Slice) + if actualResult != example.ExpectedResult { + t.Errorf("Expected stringInSlice result '%v' for example string='%v', slice='%v'. Instead got result '%v'.", example.ExpectedResult, example.String, example.Slice, actualResult) + } + } +}