k8s middleware cleanup, testcases, basic SRV (#181)

* Removing unnecessary gitignore pattern

* Updating Makefile to run unittests for subpackages

* Adding Corefile validation to ignore overlapping zones

* Fixing SRV query handling

* Updating README.md now that SRV works

* Fixing debug message, adding code comment

* Clarifying implementation of zone normalization

* "Overlapping zones" is ill-defined. Reimplemented zone overlap/subzone
  checking to contain these functions in k8s middleware and provide
  better code comments explaining the normalization.

* Separate build verbosity from test verbosity

* Cleaning up comments to match repo code style

* Merging warning messages into single message
* Moving function docs to before function declaration

* Adding test cases for k8sclient connector

* Tests cover connector create and setting base url
* Fixed bugs in connector create and setting base url functions

* Updaing README to group and order development work

* Priority focused on achieving functional parity with SkyDNS.

* Adding work items to README and cleaning up formatting

* More README format cleaning

* List formating

* Refactoring k8s API call to allow dependency injection

* Add test cases for data parsing from k8s into dataobject structures

* URL is dependency-injected to allow replacement with a mock http
  server during test execution

* Adding more data validation for JSON parsing tests

* Adding test case for GetResourceList()

* Adding notes about SkyDNS embedded IP and port record names

* Marked test case implemented.

* Fixing formatting for example command.

* Fixing formatting

* Adding notes about Docker image building.

* Adding SkyDNS work item

* Updating TODO list

* Adding name template to Corefile to specify how k8s record names are assembled

* Adding template support for multi-segment zones

* Updating example CoreFile for k8s with template comment

* Misc whitespace cleanup

* Adding SkyDNS naming notes

* Adding namespace filtering to CoreFile config

* Updating example k8sCoreFile to specify namespaces

* Removing unused codepath

* Adding check for valid namespace

* More README TODO restructuring to focus effort

* Adding template validation while parsing CoreFile

* Record name template is considered invalid if it contains a symbol of the form ${bar} where the symbol
  "${bar}" is not an accepted template symbol.

* Refactoring generation of answer records

* Parse typeName out of query string
* Refactor answer record creation as operation over list of ServiceItems

* Moving k8s API caching into SkyDNS equivalency segment

* Adding function to assemble record names from template

* Warning: This commit may be broken. Syncing to get laptop code over to dev machine.

* More todo notes

* Adding comment describing sample test data.

* Update k8sCorefile

* Adding comment

* Adding filtering support for kubernetes "type"

* Required refactoring to support reuse of the StringInSlice function.

* Cleaning up formatting

* Adding note about SkyDNS supporting word "any".

* baseUrl -> baseURL

* Also removed debug statement from core/setup/kubernetes.go

* Fixing test breaking from Url -> URL naming changes

* Changing record name template language ${...} -> {...}

* Fix formatting with go fmt

* Updating all k8sclient data getters to return error value

* Adding error message to k8sclient data accessors

* Cleaning up setup for kubernetes

* Removed verbose nils in initial k8s middleware instance
* Set reasonable defaults if CoreFile has no parameters in the
kubernetes block. (k8s endpoint, and name template)

* Formatting cleanup -- go fmt
This commit is contained in:
Michael Richmond 2016-07-07 01:40:58 -07:00 committed by Miek Gieben
parent 558c34a23e
commit 289f53d386
19 changed files with 1610 additions and 304 deletions

1
.gitignore vendored
View file

@ -2,4 +2,3 @@ query.log
Corefile Corefile
*.swp *.swp
coredns coredns
conf/devk8sCorefile

View file

@ -1,8 +1,11 @@
#VERBOSE := #BUILD_VERBOSE :=
VERBOSE := -v BUILD_VERBOSE := -v
TEST_VERBOSE :=
#TEST_VERBOSE := -v
all: all:
go build $(VERBOSE) go build $(BUILD_VERBOSE)
.PHONY: docker .PHONY: docker
docker: docker:
@ -11,11 +14,11 @@ docker:
.PHONY: deps .PHONY: deps
deps: deps:
go get go get ${BUILD_VERBOSE}
.PHONY: test .PHONY: test
test: test:
go test go test $(TEST_VERBOSE) ./...
.PHONY: clean .PHONY: clean
clean: clean:

View file

@ -4,6 +4,10 @@
kubernetes coredns.local { kubernetes coredns.local {
# Use url for k8s API endpoint # Use url for k8s API endpoint
endpoint http://localhost:8080 endpoint http://localhost:8080
# Assemble k8s record names with the template
template {service}.{namespace}.{zone}
# Only expose the k8s namespace "demo"
namespaces demo
} }
# Perform DNS response caching for the coredns.local zone # Perform DNS response caching for the coredns.local zone
# Cache timeout is provided by the integer in seconds # Cache timeout is provided by the integer in seconds

View file

@ -1,29 +1,34 @@
package setup package setup
import ( import (
// "crypto/tls" //"crypto/tls"
// "crypto/x509" //"crypto/x509"
"fmt" "fmt"
// "io/ioutil" //"io/ioutil"
// "net" //"net"
// "net/http" //"net/http"
// "time" "strings"
//"time"
"github.com/miekg/coredns/middleware" "github.com/miekg/coredns/middleware"
"github.com/miekg/coredns/middleware/kubernetes" "github.com/miekg/coredns/middleware/kubernetes"
k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient"
"github.com/miekg/coredns/middleware/kubernetes/nametemplate"
"github.com/miekg/coredns/middleware/proxy" "github.com/miekg/coredns/middleware/proxy"
// "github.com/miekg/coredns/middleware/singleflight" //"github.com/miekg/coredns/middleware/singleflight"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
const defaultK8sEndpoint = "http://localhost:8080" const (
defaultK8sEndpoint = "http://localhost:8080"
defaultNameTemplate = "{service}.{namespace}.{zone}"
)
// Kubernetes sets up the kubernetes middleware. // Kubernetes sets up the kubernetes middleware.
func Kubernetes(c *Controller) (middleware.Middleware, error) { func Kubernetes(c *Controller) (middleware.Middleware, error) {
fmt.Println("controller %v", c) fmt.Println("controller %v", c)
// TODO: Determine if subzone support required // TODO: Determine if subzone support required
kubernetes, err := kubernetesParse(c) kubernetes, err := kubernetesParse(c)
@ -39,29 +44,40 @@ func Kubernetes(c *Controller) (middleware.Middleware, error) {
func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) { func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) {
/* /*
* TODO: Remove unused state and simplify. * TODO: Remove unused state and simplify.
* Inflight and Ctx might not be needed. Leaving in place until * Inflight and Ctx might not be needed. Leaving in place until
* we take a pass at API caching and optimizing connector to the * we take a pass at API caching and optimizing connector to the
* k8s API. Single flight (or limited upper-bound) for inflight * k8s API. Single flight (or limited upper-bound) for inflight
* API calls may be desirable. * API calls may be desirable.
*/ */
k8s := kubernetes.Kubernetes{ k8s := kubernetes.Kubernetes{
Proxy: proxy.New([]string{}), Proxy: proxy.New([]string{}),
Ctx: context.Background(), Ctx: context.Background(),
// Inflight: &singleflight.Group{}, // Inflight: &singleflight.Group{},
APIConn: nil,
} }
var ( var (
endpoints = []string{defaultK8sEndpoint} endpoints = []string{defaultK8sEndpoint}
template = defaultNameTemplate
namespaces = []string{}
) )
k8s.APIConn = k8sc.NewK8sConnector(endpoints[0])
k8s.NameTemplate = new(nametemplate.NameTemplate)
k8s.NameTemplate.SetTemplate(template)
for c.Next() { for c.Next() {
if c.Val() == "kubernetes" { if c.Val() == "kubernetes" {
k8s.Zones = c.RemainingArgs() zones := c.RemainingArgs()
if len(k8s.Zones) == 0 {
if len(zones) == 0 {
k8s.Zones = c.ServerBlockHosts k8s.Zones = c.ServerBlockHosts
} else {
// Normalize requested zones
k8s.Zones = kubernetes.NormalizeZoneList(zones)
} }
middleware.Zones(k8s.Zones).FullyQualify() middleware.Zones(k8s.Zones).FullyQualify()
if c.NextBlock() { if c.NextBlock() {
// TODO(miek): 2 switches? // TODO(miek): 2 switches?
@ -72,22 +88,33 @@ func kubernetesParse(c *Controller) (kubernetes.Kubernetes, error) {
return kubernetes.Kubernetes{}, c.ArgErr() return kubernetes.Kubernetes{}, c.ArgErr()
} }
endpoints = args endpoints = args
k8s.APIConn = k8sc.NewK8sConnector(endpoints[0]) k8s.APIConn = k8sc.NewK8sConnector(endpoints[0])
} }
for c.Next() { for c.Next() {
switch c.Val() { switch c.Val() {
case "endpoint": case "template":
args := c.RemainingArgs() args := c.RemainingArgs()
if len(args) == 0 { if len(args) == 0 {
return kubernetes.Kubernetes{}, c.ArgErr() return kubernetes.Kubernetes{}, c.ArgErr()
} }
endpoints = args template = strings.Join(args, "")
err := k8s.NameTemplate.SetTemplate(template)
if err != nil {
return kubernetes.Kubernetes{}, err
}
case "namespaces":
args := c.RemainingArgs()
if len(args) == 0 {
return kubernetes.Kubernetes{}, c.ArgErr()
}
namespaces = args
k8s.Namespaces = &namespaces
} }
} }
} }
return k8s, nil return k8s, nil
} }
fmt.Println("endpoints='%v'", endpoints)
} }
fmt.Println("here before return")
return kubernetes.Kubernetes{}, nil return kubernetes.Kubernetes{}, nil
} }

View file

@ -14,7 +14,7 @@ are constructed as "myservice.mynamespace.coredns.local" where:
kubernetes [zones...] kubernetes [zones...]
~~~ ~~~
* `zones` zones kubernetes should be authorative for. * `zones` zones kubernetes should be authorative for. Overlapping zones are ignored.
~~~ ~~~
@ -88,7 +88,7 @@ The kubernetes control client can be downloaded from the generic URL:
`http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${GOOS}/${GOARCH}/${K8S_BINARY}` `http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${GOOS}/${GOARCH}/${K8S_BINARY}`
For example, the kubectl client for Linux can be downloaded using the command: For example, the kubectl client for Linux can be downloaded using the command:
`curl -sSL "http://storage.googleapis.com/kubernetes-release/release/v1.2.4/bin/linux/amd64/kubectl" `curl -sSL "http://storage.googleapis.com/kubernetes-release/release/v1.2.4/bin/linux/amd64/kubectl"`
The following `setup_kubectl.sh` script can be stored in the same directory as The following `setup_kubectl.sh` script can be stored in the same directory as
kubectl to setup kubectl to setup
@ -248,37 +248,100 @@ return the IP addresses for all services with "nginx" in the service name.
TBD: TBD:
* How does this relate the the k8s load-balancer configuration? * How does this relate the the k8s load-balancer configuration?
* Do wildcards search across namespaces? * Do wildcards search across namespaces?
* Initial implementation assumes that a namespace maps to the first DNS label below the zone managed by the kubernetes middleware. This assumption may need to be revised. * Initial implementation assumes that a namespace maps to the first DNS label
below the zone managed by the kubernetes middleware. This assumption may
need to be revised.
## TODO ## TODO
* Implement namespace filtering to different zones. * SkyDNS compatibility/equivalency:
* Implement IP selection and ordering (internal/external). * Kubernetes packaging and execution
* Implement SRV-record queries using naive lookup. * Automate packaging to allow executing in Kubernetes. That is, add Docker
* Flatten service and namespace names to valid DNS characters. (service names container build as target in Makefile. Also include anything else needed
and namespace names in k8s may use uppercase and non-DNS characters. Implement to simplify launch as the k8s DNS service.
flattening to lower case and mapping of non-DNS characters to DNS characters Note: Dockerfile already exists in coredns repo to build the docker image.
in a standard way.) This work item should identify how to pass configuration and run as a SkyDNS
* Do we need to generate synthetic zone records for namespaces? replacement.
* Implement wildcard-based lookup. * Identify any kubernetes changes necessary to use coredns as k8s DNS server. That is,
* Improve lookup to reduce size of query result obtained from k8s API. how do we consume the "--cluster-dns=" and "--cluster-domain=" arguments.
(namespace-based?, other ideas?) * Work out how to pass CoreDNS configuration via kubectl command line and yaml
* How to support label specification in Corefile to allow use of labels to service definition file.
indicate zone? (Is this even useful?) For example, the following configuration * Ensure that resolver in each kubernetes container is configured to use
exposes all services labeled for the "staging" environment and tenant "customerB" coredns instance.
in the zone "customerB.stage.local": * Update kubernetes middleware documentation to describe running CoreDNS as a
SkyDNS replacement. (Include descriptions of different ways to pass CoreFile
to coredns command.)
* Expose load-balancer IP addresses.
* Calculate SRV priority based on number of instances running.
(See SkyDNS README.md)
* Functional work
* Implement wildcard-based lookup. Minimally support `*`, consider `?` as well.
* Note from Miek on PR 181: "SkyDNS also supports the word `any`.
* Implement SkyDNS-style synthetic zones such as "svc" to group k8s objects. (This
should be optional behavior.) Also look at "pod" synthetic zones.
* Implement test cases for SkyDNS equivalent functionality.
* SkyDNS functionality, as listed in SkyDNS README: https://github.com/kubernetes/kubernetes/blob/release-1.2/cluster/addons/dns/README.md
* A records in form of `pod-ip-address.my-namespace.cluster.local`.
For example, a pod with ip `1.2.3.4` in the namespace `default`
with a dns name of `cluster.local` would have an entry:
`1-2-3-4.default.pod.cluster.local`.
* SRV records in form of
`_my-port-name._my-port-protocol.my-namespace.svc.cluster.local`
CNAME records for both regular services and headless services.
See SkyDNS README.
* A Records and hostname Based on Pod Annotations (k8s beta 1.2 feature).
See SkyDNS README.
* Note: the embedded IP and embedded port record names are weird. I
would need to know the IP/port in order to create the query to lookup
the name. Presumably these are intended for wildcard queries.
* Performance
* Improve lookup to reduce size of query result obtained from k8s API.
(namespace-based?, other ideas?)
* Caching of k8s API dataset.
* DNS response caching is good, but we should also cache at the http query
level as well. (Take a look at https://github.com/patrickmn/go-cache as
a potential expiring cache implementation for the http API queries.)
* Push notifications from k8s for data changes rather than pull via API?
* Additional features:
* Implement namespace filtering to different zones. That is, zone "a.b"
publishes services from namespace "foo", and zone "x.y" publishes services
from namespaces "bar" and "baz". (Basic version implemented -- need test cases.)
* Reverse IN-ADDR entries for services. (Is there any value in supporting
reverse lookup records?
* How to support label specification in Corefile to allow use of labels to
indicate zone? (Is this even useful?) For example, the following
configuration exposes all services labeled for the "staging" environment
and tenant "customerB" in the zone "customerB.stage.local":
~~~ kubernetes customerB.stage.local {
kubernetes customerB.stage.local { # Use url for k8s API endpoint
# Use url for k8s API endpoint endpoint http://localhost:8080
endpoint http://localhost:8080 label "environment" : "staging", "tenant" : "customerB"
label "environment" : "staging", "tenant" : "customerB" }
}
~~~
* Test with CoreDNS caching. CoreDNS caching for DNS response is working using
the `cache` directive. Tested working using 20s cache timeout and A-record queries.
* DNS response caching is good, but we should also cache at the http query
level as well. (Take a look at https://github.com/patrickmn/go-cache as
a potential expiring cache implementation for the http API queries.)
Note: label specification/selection is a killer feature for segmenting
test vs staging vs prod environments.
* Implement IP selection and ordering (internal/external). Related to
wildcards and SkyDNS use of CNAMES.
* Flatten service and namespace names to valid DNS characters. (service names
and namespace names in k8s may use uppercase and non-DNS characters. Implement
flattening to lower case and mapping of non-DNS characters to DNS characters
in a standard way.)
* Expose arbitrary kubernetes repository data as TXT records?
* Support custom user-provided templates for k8s names. A string provided
in the middleware configuration like `{service}.{namespace}.{type}` defines
the template of how to construct record names for the zone. This example
would produce `myservice.mynamespace.svc.cluster.local`. (Basic template
implemented. Need to slice zone out of current template implementation.)
* DNS Correctness
* Do we need to generate synthetic zone records for namespaces?
* Do we need to generate synthetic zone records for the skydns synthetic zones?
* Test cases
* ~~Implement test cases for http data parsing using dependency injection
for http get operations.~~
* Test with CoreDNS caching. CoreDNS caching for DNS response is working
using the `cache` directive. Tested working using 20s cache timeout
and A-record queries. Automate testing with cache in place.
* Automate CoreDNS performance tests. Initially for zone files, and for
pre-loaded k8s API cache.
* Automate integration testing with kubernetes.

View file

@ -0,0 +1,44 @@
## DNS Schema
Notes about the SkyDNS record naming scheme. (Copied from SkyDNS project README for reference while
hacking on the k8s middleware.)
### Services
#### A Records
"Normal" (not headless) Services are assigned a DNS A record for a name of the form `my-svc.my-namespace.svc.cluster.local.`
This resolves to the cluster IP of the Service.
"Headless" (without a cluster IP) Services are also assigned a DNS A record for a name of the form `my-svc.my-namespace.svc.cluster.local.`
Unlike normal Services, this resolves to the set of IPs of the pods selected by the Service.
Clients are expected to consume the set or else use standard round-robin selection from the set.
### Pods
#### A Records
When enabled, pods are assigned a DNS A record in the form of `pod-ip-address.my-namespace.pod.cluster.local.`
For example, a pod with ip `1.2.3.4` in the namespace default with a dns name of `cluster.local` would have
an entry: `1-2-3-4.default.pod.cluster.local.`
####A Records and hostname Based on Pod Annotations - A Beta Feature in Kubernetes v1.2
Currently when a pod is created, its hostname is the Pod's `metadata.name` value.
With v1.2, users can specify a Pod annotation, `pod.beta.kubernetes.io/hostname`, to specify what the Pod's hostname should be.
If the annotation is specified, the annotation value takes precendence over the Pod's name, to be the hostname of the pod.
For example, given a Pod with annotation `pod.beta.kubernetes.io/hostname: my-pod-name`, the Pod will have its hostname set to "my-pod-name".
v1.2 introduces a beta feature where the user can specify a Pod annotation, `pod.beta.kubernetes.io/subdomain`, to specify what the Pod's subdomain should be.
If the annotation is specified, the fully qualified Pod hostname will be "<hostname>.<subdomain>.<pod namespace>.svc.<cluster domain>".
For example, given a Pod with the hostname annotation set to "foo", and the subdomain annotation set to "bar", in namespace "my-namespace", the pod will set its own FQDN as "foo.bar.my-namespace.svc.cluster.local"
If there exists a headless service in the same namespace as the pod and with the same name as the subdomain, the cluster's KubeDNS Server will also return an A record for the Pod's fully qualified hostname.
Given a Pod with the hostname annotation set to "foo" and the subdomain annotation set to "bar", and a headless Service named "bar" in the same namespace, the pod will see it's own FQDN as "foo.bar.my-namespace.svc.cluster.local". DNS will serve an A record at that name, pointing to the Pod's IP.
With v1.2, the Endpoints object also has a new annotation `endpoints.beta.kubernetes.io/hostnames-map`. Its value is the json representation of map[string(IP)][endpoints.HostRecord], for example: '{"10.245.1.6":{HostName: "my-webserver"}}'.
If the Endpoints are for a headless service, then A records will be created with the format <hostname>.<service name>.<pod namespace>.svc.<cluster domain>
For the example json, if endpoints are for a headless service named "bar", and one of the endpoints has IP "10.245.1.6", then a A record will be created with the name "my-webserver.bar.my-namespace.svc.cluster.local" and the A record lookup would return "10.245.1.6".
This endpoints annotation generally does not need to be specified by end-users, but can used by the internal service controller to deliver the aforementioned feature.

View file

@ -11,15 +11,15 @@ import (
func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) { func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
fmt.Println("[debug] here entering ServeDNS: ctx:%v dnsmsg:%v", ctx, r) fmt.Printf("[debug] here entering ServeDNS: ctx:%v dnsmsg:%v\n", ctx, r)
state := middleware.State{W: w, Req: r} state := middleware.State{W: w, Req: r}
if state.QClass() != dns.ClassINET { if state.QClass() != dns.ClassINET {
return dns.RcodeServerFailure, fmt.Errorf("can only deal with ClassINET") return dns.RcodeServerFailure, fmt.Errorf("can only deal with ClassINET")
} }
// Check that query matches one of the zones served by this middleware, // Check that query matches one of the zones served by this middleware,
// otherwise delegate to the next in the pipeline. // otherwise delegate to the next in the pipeline.
zone := middleware.Zones(k.Zones).Matches(state.Name()) zone := middleware.Zones(k.Zones).Matches(state.Name())
if zone == "" { if zone == "" {
if k.Next == nil { if k.Next == nil {
@ -43,6 +43,9 @@ func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.M
records, err = k.AAAA(zone, state, nil) records, err = k.AAAA(zone, state, nil)
case "TXT": case "TXT":
records, err = k.TXT(zone, state) records, err = k.TXT(zone, state)
// TODO: change lookup to return appropriate error. Then add code below
// this switch to check for the error and return not implemented.
//return dns.RcodeNotImplemented, nil
case "CNAME": case "CNAME":
records, err = k.CNAME(zone, state) records, err = k.CNAME(zone, state)
case "MX": case "MX":

View file

@ -1,110 +1,113 @@
package k8sclient package k8sclient
import ( import (
"encoding/json" "encoding/json"
"net/http" "net/http"
) )
// getK8sAPIResponse wraps the http.Get(url) function to provide dependency
func getJson(url string, target interface{}) error { // injection for unit testing.
r, err := http.Get(url) var getK8sAPIResponse = func(url string) (resp *http.Response, err error) {
if err != nil { resp, err = http.Get(url)
return err return resp, err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
} }
func parseJson(url string, target interface{}) error {
r, err := getK8sAPIResponse(url)
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
// Kubernetes Resource List // Kubernetes Resource List
type ResourceList struct { type ResourceList struct {
Kind string `json:"kind"` Kind string `json:"kind"`
GroupVersion string `json:"groupVersion"` GroupVersion string `json:"groupVersion"`
Resources []resource `json:"resources"` Resources []resource `json:"resources"`
} }
type resource struct { type resource struct {
Name string `json:"name"` Name string `json:"name"`
Namespaced bool `json:"namespaced"` Namespaced bool `json:"namespaced"`
Kind string `json:"kind"` Kind string `json:"kind"`
} }
// Kubernetes NamespaceList // Kubernetes NamespaceList
type NamespaceList struct { type NamespaceList struct {
Kind string `json:"kind"` Kind string `json:"kind"`
APIVersion string `json:"apiVersion"` APIVersion string `json:"apiVersion"`
Metadata apiListMetadata `json:"metadata"` Metadata apiListMetadata `json:"metadata"`
Items []nsItems `json:"items"` Items []nsItems `json:"items"`
} }
type apiListMetadata struct { type apiListMetadata struct {
SelfLink string `json:"selfLink"` SelfLink string `json:"selfLink"`
resourceVersion string `json:"resourceVersion"` ResourceVersion string `json:"resourceVersion"`
} }
type nsItems struct { type nsItems struct {
Metadata nsMetadata `json:"metadata"` Metadata nsMetadata `json:"metadata"`
Spec nsSpec `json:"spec"` Spec nsSpec `json:"spec"`
Status nsStatus `json:"status"` Status nsStatus `json:"status"`
} }
type nsMetadata struct { type nsMetadata struct {
Name string `json:"name"` Name string `json:"name"`
SelfLink string `json:"selfLink"` SelfLink string `json:"selfLink"`
Uid string `json:"uid"` Uid string `json:"uid"`
ResourceVersion string `json:"resourceVersion"` ResourceVersion string `json:"resourceVersion"`
CreationTimestamp string `json:"creationTimestamp"` CreationTimestamp string `json:"creationTimestamp"`
} }
type nsSpec struct { type nsSpec struct {
Finalizers []string `json:"finalizers"` Finalizers []string `json:"finalizers"`
} }
type nsStatus struct { type nsStatus struct {
Phase string `json:"phase"` Phase string `json:"phase"`
} }
// Kubernetes ServiceList // Kubernetes ServiceList
type ServiceList struct { type ServiceList struct {
Kind string `json:"kind"` Kind string `json:"kind"`
APIVersion string `json:"apiVersion"` APIVersion string `json:"apiVersion"`
Metadata apiListMetadata `json:"metadata"` Metadata apiListMetadata `json:"metadata"`
Items []ServiceItem `json:"items"` Items []ServiceItem `json:"items"`
} }
type ServiceItem struct { type ServiceItem struct {
Metadata serviceMetadata `json:"metadata"` Metadata serviceMetadata `json:"metadata"`
Spec serviceSpec `json:"spec"` Spec serviceSpec `json:"spec"`
// Status serviceStatus `json:"status"` // Status serviceStatus `json:"status"`
} }
type serviceMetadata struct { type serviceMetadata struct {
Name string `json:"name"` Name string `json:"name"`
Namespace string `json:"namespace"` Namespace string `json:"namespace"`
SelfLink string `json:"selfLink"` SelfLink string `json:"selfLink"`
Uid string `json:"uid"` Uid string `json:"uid"`
ResourceVersion string `json:"resourceVersion"` ResourceVersion string `json:"resourceVersion"`
CreationTimestamp string `json:"creationTimestamp"` CreationTimestamp string `json:"creationTimestamp"`
// labels // labels
} }
type serviceSpec struct { type serviceSpec struct {
Ports []servicePort `json:"ports"` Ports []servicePort `json:"ports"`
ClusterIP string `json:"clusterIP"` ClusterIP string `json:"clusterIP"`
Type string `json:"type"` Type string `json:"type"`
SessionAffinity string `json:"sessionAffinity"` SessionAffinity string `json:"sessionAffinity"`
} }
type servicePort struct { type servicePort struct {
Name string `json:"name"` Name string `json:"name"`
Protocol string `json:"protocol"` Protocol string `json:"protocol"`
Port int `json:"port"` Port int `json:"port"`
TargetPort int `json:"targetPort"` TargetPort int `json:"targetPort"`
} }
type serviceStatus struct { type serviceStatus struct {
LoadBalancer string `json:"loadBalancer"` LoadBalancer string `json:"loadBalancer"`
} }

View file

@ -1,117 +1,157 @@
package k8sclient package k8sclient
import ( import (
// "fmt" "errors"
"net/url" "fmt"
"net/url"
"strings"
) )
// API strings // API strings
const ( const (
apiBase = "/api/v1" apiBase = "/api/v1"
apiNamespaces = "/namespaces" apiNamespaces = "/namespaces"
apiServices = "/services" apiServices = "/services"
) )
// Defaults // Defaults
const ( const (
defaultBaseUrl = "http://localhost:8080" defaultBaseURL = "http://localhost:8080"
) )
type K8sConnector struct { type K8sConnector struct {
baseUrl string baseURL string
} }
func (c *K8sConnector) SetBaseUrl(u string) error { func (c *K8sConnector) SetBaseURL(u string) error {
validUrl, error := url.Parse(u) url, error := url.Parse(u)
if error != nil { if error != nil {
return error return error
} }
c.baseUrl = validUrl.String()
return nil if !url.IsAbs() {
return errors.New("k8sclient: Kubernetes endpoint url must be an absolute URL")
}
c.baseURL = url.String()
return nil
} }
func (c *K8sConnector) GetBaseUrl() string { func (c *K8sConnector) GetBaseURL() string {
return c.baseUrl return c.baseURL
} }
// URL constructor separated from code to support dependency injection
func (c *K8sConnector) GetResourceList() *ResourceList { // for unit tests.
resources := new(ResourceList) var makeURL = func(parts []string) string {
return strings.Join(parts, "")
error := getJson((c.baseUrl + apiBase), resources)
if error != nil {
return nil
}
return resources
} }
func (c *K8sConnector) GetResourceList() (*ResourceList, error) {
resources := new(ResourceList)
func (c *K8sConnector) GetNamespaceList() *NamespaceList { url := makeURL([]string{c.baseURL, apiBase})
namespaces := new(NamespaceList) err := parseJson(url, resources)
// TODO: handle no response from k8s
if err != nil {
fmt.Printf("[ERROR] Response from kubernetes API for GetResourceList() is: %v\n", err)
return nil, err
}
error := getJson((c.baseUrl + apiBase + apiNamespaces), namespaces) return resources, nil
if error != nil {
return nil
}
return namespaces
} }
func (c *K8sConnector) GetNamespaceList() (*NamespaceList, error) {
namespaces := new(NamespaceList)
func (c *K8sConnector) GetServiceList() *ServiceList { url := makeURL([]string{c.baseURL, apiBase, apiNamespaces})
services := new(ServiceList) err := parseJson(url, namespaces)
if err != nil {
fmt.Printf("[ERROR] Response from kubernetes API for GetNamespaceList() is: %v\n", err)
return nil, err
}
error := getJson((c.baseUrl + apiBase + apiServices), services) return namespaces, nil
if error != nil {
return nil
}
return services
} }
func (c *K8sConnector) GetServiceList() (*ServiceList, error) {
services := new(ServiceList)
func (c *K8sConnector) GetServicesByNamespace() map[string][]ServiceItem { url := makeURL([]string{c.baseURL, apiBase, apiServices})
// GetServicesByNamespace returns a map of namespacename :: [ kubernetesServiceItem ] err := parseJson(url, services)
// TODO: handle no response from k8s
if err != nil {
fmt.Printf("[ERROR] Response from kubernetes API for GetServiceList() is: %v\n", err)
return nil, err
}
items := make(map[string][]ServiceItem) return services, nil
k8sServiceList := c.GetServiceList()
k8sItemList := k8sServiceList.Items
for _, i := range k8sItemList {
namespace := i.Metadata.Namespace
items[namespace] = append(items[namespace], i)
}
return items
} }
// GetServicesByNamespace returns a map of
// namespacename :: [ kubernetesServiceItem ]
func (c *K8sConnector) GetServicesByNamespace() (map[string][]ServiceItem, error) {
func (c *K8sConnector) GetServiceItemInNamespace(namespace string, servicename string) *ServiceItem { items := make(map[string][]ServiceItem)
// GetServiceItemInNamespace returns the ServiceItem that matches servicename in the namespace
itemMap := c.GetServicesByNamespace() k8sServiceList, err := c.GetServiceList()
// TODO: Handle case where namesapce == nil if err != nil {
fmt.Printf("[ERROR] Getting service list produced error: %v", err)
return nil, err
}
for _, x := range itemMap[namespace] { // TODO: handle no response from k8s
if x.Metadata.Name == servicename { if k8sServiceList == nil {
return &x return nil, nil
} }
}
// No matching item found in namespace k8sItemList := k8sServiceList.Items
return nil
for _, i := range k8sItemList {
namespace := i.Metadata.Namespace
items[namespace] = append(items[namespace], i)
}
return items, nil
} }
// GetServiceItemsInNamespace returns the ServiceItems that match
// servicename in the namespace
func (c *K8sConnector) GetServiceItemsInNamespace(namespace string, servicename string) ([]*ServiceItem, error) {
func NewK8sConnector(baseurl string) *K8sConnector { itemMap, err := c.GetServicesByNamespace()
k := new(K8sConnector)
k.SetBaseUrl(baseurl)
return k if err != nil {
fmt.Printf("[ERROR] Getting service list produced error: %v", err)
return nil, err
}
// TODO: Handle case where namespace == nil
var serviceItems []*ServiceItem
for _, x := range itemMap[namespace] {
if x.Metadata.Name == servicename {
serviceItems = append(serviceItems, &x)
}
}
return serviceItems, nil
}
func NewK8sConnector(baseURL string) *K8sConnector {
k := new(K8sConnector)
if baseURL == "" {
baseURL = defaultBaseURL
}
err := k.SetBaseURL(baseURL)
if err != nil {
return nil
}
return k
} }

View file

@ -0,0 +1,680 @@
package k8sclient
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
)
var validURLs = []string{
"http://www.github.com",
"http://www.github.com:8080",
"http://8.8.8.8",
"http://8.8.8.8:9090",
"www.github.com:8080",
}
var invalidURLs = []string{
"www.github.com",
"8.8.8.8",
"8.8.8.8:1010",
"8.8`8.8",
}
func TestNewK8sConnector(t *testing.T) {
var conn *K8sConnector
var url string
// Create with empty URL
conn = nil
url = ""
conn = NewK8sConnector("")
if conn == nil {
t.Errorf("Expected K8sConnector instance. Instead got '%v'", conn)
}
url = conn.GetBaseURL()
if url != defaultBaseURL {
t.Errorf("Expected K8sConnector instance to be initialized with defaultBaseURL. Instead got '%v'", url)
}
// Create with valid URL
for _, validURL := range validURLs {
conn = nil
url = ""
conn = NewK8sConnector(validURL)
if conn == nil {
t.Errorf("Expected K8sConnector instance. Instead got '%v'", conn)
}
url = conn.GetBaseURL()
if url != validURL {
t.Errorf("Expected K8sConnector instance to be initialized with supplied url '%v'. Instead got '%v'", validURL, url)
}
}
// Create with invalid URL
for _, invalidURL := range invalidURLs {
conn = nil
url = ""
conn = NewK8sConnector(invalidURL)
if conn != nil {
t.Errorf("Expected to not get K8sConnector instance. Instead got '%v'", conn)
continue
}
}
}
func TestSetBaseURL(t *testing.T) {
// SetBaseURL with valid URLs should work...
for _, validURL := range validURLs {
conn := NewK8sConnector(defaultBaseURL)
err := conn.SetBaseURL(validURL)
if err != nil {
t.Errorf("Expected to receive nil, instead got error '%v'", err)
continue
}
url := conn.GetBaseURL()
if url != validURL {
t.Errorf("Expected to connector url to be set to value '%v', instead set to '%v'", validURL, url)
continue
}
}
// SetBaseURL with invalid or non absolute URLs should not change state...
for _, invalidURL := range invalidURLs {
conn := NewK8sConnector(defaultBaseURL)
originalURL := conn.GetBaseURL()
err := conn.SetBaseURL(invalidURL)
if err == nil {
t.Errorf("Expected to receive an error value, instead got nil")
}
url := conn.GetBaseURL()
if url != originalURL {
t.Errorf("Expected base url to not change, instead it changed to '%v'", url)
}
}
}
func TestGetNamespaceList(t *testing.T) {
// Set up a test http server
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, namespaceListJsonData)
}))
defer testServer.Close()
// Overwrite URL constructor to access testServer
makeURL = func(parts []string) string {
return testServer.URL
}
expectedNamespaces := []string{"default", "demo", "test"}
apiConn := NewK8sConnector("")
namespaceList, err := apiConn.GetNamespaceList()
if err != nil {
t.Errorf("Expected no error from from GetNamespaceList(), instead got %v", err)
}
if namespaceList == nil {
t.Errorf("Expected data from GetNamespaceList(), instead got nil")
}
kind := namespaceList.Kind
if kind != "NamespaceList" {
t.Errorf("Expected data from GetNamespaceList() to have Kind='NamespaceList', instead got Kind='%v'", kind)
}
// Ensure correct number of namespaces found
expectedCount := len(expectedNamespaces)
namespaceCount := len(namespaceList.Items)
if namespaceCount != expectedCount {
t.Errorf("Expected '%v' namespaces from GetNamespaceList(), instead found '%v' namespaces", expectedCount, namespaceCount)
}
// Check that all expectedNamespaces are found in the parsed data
for _, ns := range expectedNamespaces {
found := false
for _, item := range namespaceList.Items {
if item.Metadata.Name == ns {
found = true
break
}
}
if !found {
t.Errorf("Expected '%v' namespace is not in the parsed data from GetServicesByNamespace()", ns)
}
}
}
func TestGetServiceList(t *testing.T) {
// Set up a test http server
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, serviceListJsonData)
}))
defer testServer.Close()
// Overwrite URL constructor to access testServer
makeURL = func(parts []string) string {
return testServer.URL
}
expectedServices := []string{"kubernetes", "mynginx", "mywebserver"}
apiConn := NewK8sConnector("")
serviceList, err := apiConn.GetServiceList()
if err != nil {
t.Errorf("Expected no error from from GetNamespaceList(), instead got %v", err)
}
if serviceList == nil {
t.Errorf("Expected data from GetServiceList(), instead got nil")
}
kind := serviceList.Kind
if kind != "ServiceList" {
t.Errorf("Expected data from GetServiceList() to have Kind='ServiceList', instead got Kind='%v'", kind)
}
// Ensure correct number of services found
expectedCount := len(expectedServices)
serviceCount := len(serviceList.Items)
if serviceCount != expectedCount {
t.Errorf("Expected '%v' services from GetServiceList(), instead found '%v' services", expectedCount, serviceCount)
}
// Check that all expectedServices are found in the parsed data
for _, s := range expectedServices {
found := false
for _, item := range serviceList.Items {
if item.Metadata.Name == s {
found = true
break
}
}
if !found {
t.Errorf("Expected '%v' service is not in the parsed data from GetServiceList()", s)
}
}
}
func TestGetServicesByNamespace(t *testing.T) {
// Set up a test http server
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, serviceListJsonData)
}))
defer testServer.Close()
// Overwrite URL constructor to access testServer
makeURL = func(parts []string) string {
return testServer.URL
}
expectedNamespaces := []string{"default", "demo"}
apiConn := NewK8sConnector("")
servicesByNamespace, err := apiConn.GetServicesByNamespace()
if err != nil {
t.Errorf("Expected no error from from GetServicesByNamespace(), instead got %v", err)
}
// Ensure correct number of namespaces found
expectedCount := len(expectedNamespaces)
namespaceCount := len(servicesByNamespace)
if namespaceCount != expectedCount {
t.Errorf("Expected '%v' namespaces from GetServicesByNamespace(), instead found '%v' namespaces", expectedCount, namespaceCount)
}
// Check that all expectedNamespaces are found in the parsed data
for _, ns := range expectedNamespaces {
_, ok := servicesByNamespace[ns]
if !ok {
t.Errorf("Expected '%v' namespace is not in the parsed data from GetServicesByNamespace()", ns)
}
}
}
func TestGetResourceList(t *testing.T) {
// Set up a test http server
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, resourceListJsonData)
}))
defer testServer.Close()
// Overwrite URL constructor to access testServer
makeURL = func(parts []string) string {
return testServer.URL
}
expectedResources := []string{"bindings",
"componentstatuses",
"configmaps",
"endpoints",
"events",
"limitranges",
"namespaces",
"namespaces/finalize",
"namespaces/status",
"nodes",
"nodes/proxy",
"nodes/status",
"persistentvolumeclaims",
"persistentvolumeclaims/status",
"persistentvolumes",
"persistentvolumes/status",
"pods",
"pods/attach",
"pods/binding",
"pods/exec",
"pods/log",
"pods/portforward",
"pods/proxy",
"pods/status",
"podtemplates",
"replicationcontrollers",
"replicationcontrollers/scale",
"replicationcontrollers/status",
"resourcequotas",
"resourcequotas/status",
"secrets",
"serviceaccounts",
"services",
"services/proxy",
"services/status",
}
apiConn := NewK8sConnector("")
resourceList, err := apiConn.GetResourceList()
if err != nil {
t.Errorf("Expected no error from from GetResourceList(), instead got %v", err)
}
if resourceList == nil {
t.Errorf("Expected data from GetResourceList(), instead got nil")
}
kind := resourceList.Kind
if kind != "APIResourceList" {
t.Errorf("Expected data from GetResourceList() to have Kind='ResourceList', instead got Kind='%v'", kind)
}
// Ensure correct number of resources found
expectedCount := len(expectedResources)
resourceCount := len(resourceList.Resources)
if resourceCount != expectedCount {
t.Errorf("Expected '%v' resources from GetResourceList(), instead found '%v' resources", expectedCount, resourceCount)
}
// Check that all expectedResources are found in the parsed data
for _, r := range expectedResources {
found := false
for _, item := range resourceList.Resources {
if item.Name == r {
found = true
break
}
}
if !found {
t.Errorf("Expected '%v' resource is not in the parsed data from GetResourceList()", r)
}
}
}
// Sample namespace data for kubernetes with 3 namespaces:
// "default", "demo", and "test".
const namespaceListJsonData string = `{
"kind": "NamespaceList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/namespaces/",
"resourceVersion": "121279"
},
"items": [
{
"metadata": {
"name": "default",
"selfLink": "/api/v1/namespaces/default",
"uid": "fb1c92d1-2f39-11e6-b9db-0800279930f6",
"resourceVersion": "6",
"creationTimestamp": "2016-06-10T18:34:35Z"
},
"spec": {
"finalizers": [
"kubernetes"
]
},
"status": {
"phase": "Active"
}
},
{
"metadata": {
"name": "demo",
"selfLink": "/api/v1/namespaces/demo",
"uid": "73be8ffd-2f3a-11e6-b9db-0800279930f6",
"resourceVersion": "111",
"creationTimestamp": "2016-06-10T18:37:57Z"
},
"spec": {
"finalizers": [
"kubernetes"
]
},
"status": {
"phase": "Active"
}
},
{
"metadata": {
"name": "test",
"selfLink": "/api/v1/namespaces/test",
"uid": "c0be05fa-3352-11e6-b9db-0800279930f6",
"resourceVersion": "121276",
"creationTimestamp": "2016-06-15T23:41:59Z"
},
"spec": {
"finalizers": [
"kubernetes"
]
},
"status": {
"phase": "Active"
}
}
]
}`
// Sample service data for kubernetes with 3 services:
// * "kubernetes" (in "default" namespace)
// * "mynginx" (in "demo" namespace)
// * "webserver" (in "demo" namespace)
const serviceListJsonData string = `
{
"kind": "ServiceList",
"apiVersion": "v1",
"metadata": {
"selfLink": "/api/v1/services",
"resourceVersion": "147965"
},
"items": [
{
"metadata": {
"name": "kubernetes",
"namespace": "default",
"selfLink": "/api/v1/namespaces/default/services/kubernetes",
"uid": "fb1cb0d3-2f39-11e6-b9db-0800279930f6",
"resourceVersion": "7",
"creationTimestamp": "2016-06-10T18:34:35Z",
"labels": {
"component": "apiserver",
"provider": "kubernetes"
}
},
"spec": {
"ports": [
{
"name": "https",
"protocol": "TCP",
"port": 443,
"targetPort": 443
}
],
"clusterIP": "10.0.0.1",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
},
{
"metadata": {
"name": "mynginx",
"namespace": "demo",
"selfLink": "/api/v1/namespaces/demo/services/mynginx",
"uid": "93c117ac-2f3a-11e6-b9db-0800279930f6",
"resourceVersion": "147",
"creationTimestamp": "2016-06-10T18:38:51Z",
"labels": {
"run": "mynginx"
}
},
"spec": {
"ports": [
{
"protocol": "TCP",
"port": 80,
"targetPort": 80
}
],
"selector": {
"run": "mynginx"
},
"clusterIP": "10.0.0.132",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
},
{
"metadata": {
"name": "mywebserver",
"namespace": "demo",
"selfLink": "/api/v1/namespaces/demo/services/mywebserver",
"uid": "aed62187-33e5-11e6-a224-0800279930f6",
"resourceVersion": "138185",
"creationTimestamp": "2016-06-16T17:13:45Z",
"labels": {
"run": "mywebserver"
}
},
"spec": {
"ports": [
{
"protocol": "TCP",
"port": 443,
"targetPort": 443
}
],
"selector": {
"run": "mywebserver"
},
"clusterIP": "10.0.0.63",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
}
]
}
`
// Sample resource data for kubernetes.
const resourceListJsonData string = `{
"kind": "APIResourceList",
"groupVersion": "v1",
"resources": [
{
"name": "bindings",
"namespaced": true,
"kind": "Binding"
},
{
"name": "componentstatuses",
"namespaced": false,
"kind": "ComponentStatus"
},
{
"name": "configmaps",
"namespaced": true,
"kind": "ConfigMap"
},
{
"name": "endpoints",
"namespaced": true,
"kind": "Endpoints"
},
{
"name": "events",
"namespaced": true,
"kind": "Event"
},
{
"name": "limitranges",
"namespaced": true,
"kind": "LimitRange"
},
{
"name": "namespaces",
"namespaced": false,
"kind": "Namespace"
},
{
"name": "namespaces/finalize",
"namespaced": false,
"kind": "Namespace"
},
{
"name": "namespaces/status",
"namespaced": false,
"kind": "Namespace"
},
{
"name": "nodes",
"namespaced": false,
"kind": "Node"
},
{
"name": "nodes/proxy",
"namespaced": false,
"kind": "Node"
},
{
"name": "nodes/status",
"namespaced": false,
"kind": "Node"
},
{
"name": "persistentvolumeclaims",
"namespaced": true,
"kind": "PersistentVolumeClaim"
},
{
"name": "persistentvolumeclaims/status",
"namespaced": true,
"kind": "PersistentVolumeClaim"
},
{
"name": "persistentvolumes",
"namespaced": false,
"kind": "PersistentVolume"
},
{
"name": "persistentvolumes/status",
"namespaced": false,
"kind": "PersistentVolume"
},
{
"name": "pods",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/attach",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/binding",
"namespaced": true,
"kind": "Binding"
},
{
"name": "pods/exec",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/log",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/portforward",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/proxy",
"namespaced": true,
"kind": "Pod"
},
{
"name": "pods/status",
"namespaced": true,
"kind": "Pod"
},
{
"name": "podtemplates",
"namespaced": true,
"kind": "PodTemplate"
},
{
"name": "replicationcontrollers",
"namespaced": true,
"kind": "ReplicationController"
},
{
"name": "replicationcontrollers/scale",
"namespaced": true,
"kind": "Scale"
},
{
"name": "replicationcontrollers/status",
"namespaced": true,
"kind": "ReplicationController"
},
{
"name": "resourcequotas",
"namespaced": true,
"kind": "ResourceQuota"
},
{
"name": "resourcequotas/status",
"namespaced": true,
"kind": "ResourceQuota"
},
{
"name": "secrets",
"namespaced": true,
"kind": "Secret"
},
{
"name": "serviceaccounts",
"namespaced": true,
"kind": "ServiceAccount"
},
{
"name": "services",
"namespaced": true,
"kind": "Service"
},
{
"name": "services/proxy",
"namespaced": true,
"kind": "Service"
},
{
"name": "services/status",
"namespaced": true,
"kind": "Service"
}
]
}`

View file

@ -2,111 +2,148 @@
package kubernetes package kubernetes
import ( import (
"fmt" "errors"
"strings" "fmt"
"time" "time"
"github.com/miekg/coredns/middleware" "github.com/miekg/coredns/middleware"
"github.com/miekg/coredns/middleware/kubernetes/msg"
k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient" k8sc "github.com/miekg/coredns/middleware/kubernetes/k8sclient"
"github.com/miekg/coredns/middleware/kubernetes/msg"
"github.com/miekg/coredns/middleware/kubernetes/nametemplate"
"github.com/miekg/coredns/middleware/kubernetes/util"
"github.com/miekg/coredns/middleware/proxy" "github.com/miekg/coredns/middleware/proxy"
// "github.com/miekg/coredns/middleware/singleflight" // "github.com/miekg/coredns/middleware/singleflight"
"github.com/miekg/dns" "github.com/miekg/dns"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
type Kubernetes struct { type Kubernetes struct {
Next middleware.Handler Next middleware.Handler
Zones []string Zones []string
Proxy proxy.Proxy // Proxy for looking up names during the resolution process Proxy proxy.Proxy // Proxy for looking up names during the resolution process
Ctx context.Context Ctx context.Context
// Inflight *singleflight.Group // Inflight *singleflight.Group
APIConn *k8sc.K8sConnector APIConn *k8sc.K8sConnector
NameTemplate *nametemplate.NameTemplate
Namespaces *[]string
} }
// getZoneForName returns the zone string that matches the name and a
// list of the DNS labels from name that are within the zone.
// For example, if "coredns.local" is a zone configured for the
// Kubernetes middleware, then getZoneForName("a.b.coredns.local")
// will return ("coredns.local", ["a", "b"]).
func (g Kubernetes) getZoneForName(name string) (string, []string) { func (g Kubernetes) getZoneForName(name string) (string, []string) {
/* var zone string
* getZoneForName returns the zone string that matches the name and a var serviceSegments []string
* list of the DNS labels from name that are within the zone.
* For example, if "coredns.local" is a zone configured for the
* Kubernetes middleware, then getZoneForName("a.b.coredns.local")
* will return ("coredns.local", ["a", "b"]).
*/
var zone string
var serviceSegments []string
for _, z := range g.Zones { for _, z := range g.Zones {
if dns.IsSubDomain(z, name) { if dns.IsSubDomain(z, name) {
zone = z zone = z
serviceSegments = dns.SplitDomainName(name)
serviceSegments = serviceSegments[:len(serviceSegments) - dns.CountLabel(zone)]
break
}
}
return zone, serviceSegments serviceSegments = dns.SplitDomainName(name)
} serviceSegments = serviceSegments[:len(serviceSegments)-dns.CountLabel(zone)]
break
}
}
return zone, serviceSegments
}
// Records looks up services in kubernetes. // Records looks up services in kubernetes.
// If exact is true, it will lookup just // If exact is true, it will lookup just
// this name. This is used when find matches when completing SRV lookups // this name. This is used when find matches when completing SRV lookups
// for instance. // for instance.
func (g Kubernetes) Records(name string, exact bool) ([]msg.Service, error) { func (g Kubernetes) Records(name string, exact bool) ([]msg.Service, error) {
var (
serviceName string
namespace string
typeName string
)
fmt.Println("enter Records('", name, "', ", exact, ")") fmt.Println("enter Records('", name, "', ", exact, ")")
zone, serviceSegments := g.getZoneForName(name)
zone, serviceSegments := g.getZoneForName(name) /*
// For initial implementation, assume namespace is first serviceSegment
// and service name is remaining segments.
serviceSegLen := len(serviceSegments)
if serviceSegLen >= 2 {
namespace = serviceSegments[serviceSegLen-1]
serviceName = strings.Join(serviceSegments[:serviceSegLen-1], ".")
}
// else we are looking up the zone. So handle the NS, SOA records etc.
*/
var serviceName string // TODO: Implementation above globbed together segments for the serviceName if
var namespace string // multiple segments remained. Determine how to do similar globbing using
// the template-based implementation.
namespace = g.NameTemplate.GetNamespaceFromSegmentArray(serviceSegments)
serviceName = g.NameTemplate.GetServiceFromSegmentArray(serviceSegments)
typeName = g.NameTemplate.GetTypeFromSegmentArray(serviceSegments)
// For initial implementation, assume namespace is first serviceSegment fmt.Println("[debug] exact: ", exact)
// and service name is remaining segments. fmt.Println("[debug] zone: ", zone)
serviceSegLen := len(serviceSegments) fmt.Println("[debug] servicename: ", serviceName)
if serviceSegLen >= 2 { fmt.Println("[debug] namespace: ", namespace)
namespace = serviceSegments[serviceSegLen-1] fmt.Println("[debug] typeName: ", typeName)
serviceName = strings.Join(serviceSegments[:serviceSegLen-1], ".") fmt.Println("[debug] APIconn: ", g.APIConn)
}
// else we are looking up the zone. So handle the NS, SOA records etc.
fmt.Println("[debug] zone: ", zone) // TODO: Implement wildcard support to allow blank namespace value
fmt.Println("[debug] servicename: ", serviceName) if namespace == "" {
fmt.Println("[debug] namespace: ", namespace) err := errors.New("Parsing query string did not produce a namespace value")
fmt.Println("[debug] APIconn: ", g.APIConn) fmt.Printf("[ERROR] %v\n", err)
return nil, err
}
k8sItem := g.APIConn.GetServiceItemInNamespace(namespace, serviceName) // Abort if the namespace is not published per CoreFile
fmt.Println("[debug] k8s item:", k8sItem) if g.Namespaces != nil && !util.StringInSlice(namespace, *g.Namespaces) {
return nil, nil
}
switch { k8sItems, err := g.APIConn.GetServiceItemsInNamespace(namespace, serviceName)
case exact && k8sItem == nil: fmt.Println("[debug] k8s items:", k8sItems)
fmt.Println("here2")
return nil, nil
}
if k8sItem == nil { if err != nil {
// Did not find item in k8s fmt.Printf("[ERROR] Got error while looking up ServiceItems. Error is: %v\n", err)
return nil, nil return nil, err
} }
if k8sItems == nil {
// Did not find item in k8s
return nil, nil
}
fmt.Println("[debug] clusterIP:", k8sItem.Spec.ClusterIP) // test := g.NameTemplate.GetRecordNameFromNameValues(nametemplate.NameValues{ServiceName: serviceName, TypeName: typeName, Namespace: namespace, Zone: zone})
// fmt.Printf("[debug] got recordname %v\n", test)
for _, p := range k8sItem.Spec.Ports { records := g.getRecordsForServiceItems(k8sItems, name)
fmt.Println("[debug] host:", name)
fmt.Println("[debug] port:", p.Port)
}
clusterIP := k8sItem.Spec.ClusterIP return records, nil
var records []msg.Service }
for _, p := range k8sItem.Spec.Ports{
s := msg.Service{Host: clusterIP, Port: p.Port}
records = append(records, s)
}
return records, nil // TODO: assemble name from parts found in k8s data based on name template rather than reusing query string
func (g Kubernetes) getRecordsForServiceItems(serviceItems []*k8sc.ServiceItem, name string) []msg.Service {
var records []msg.Service
for _, item := range serviceItems {
fmt.Println("[debug] clusterIP:", item.Spec.ClusterIP)
for _, p := range item.Spec.Ports {
fmt.Println("[debug] port:", p.Port)
}
clusterIP := item.Spec.ClusterIP
s := msg.Service{Host: name}
records = append(records, s)
for _, p := range item.Spec.Ports {
s := msg.Service{Host: clusterIP, Port: p.Port}
records = append(records, s)
}
}
fmt.Printf("[debug] records from getRecordsForServiceItems(): %v\n", records)
return records
} }
/* /*
@ -121,13 +158,13 @@ func (g Kubernetes) Get(path string, recursive bool) (bool, error) {
*/ */
func (g Kubernetes) splitDNSName(name string) []string { func (g Kubernetes) splitDNSName(name string) []string {
l := dns.SplitDomainName(name) l := dns.SplitDomainName(name)
for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
l[i], l[j] = l[j], l[i] l[i], l[j] = l[j], l[i]
} }
return l return l
} }
// skydns/local/skydns/east/staging/web // skydns/local/skydns/east/staging/web
@ -215,9 +252,9 @@ func isKubernetesNameError(err error) bool {
} }
const ( const (
priority = 10 // default priority when nothing is set priority = 10 // default priority when nothing is set
ttl = 300 // default ttl when nothing is set ttl = 300 // default ttl when nothing is set
minTtl = 60 minTtl = 60
hostmaster = "hostmaster" hostmaster = "hostmaster"
k8sTimeout = 5 * time.Second k8sTimeout = 5 * time.Second
) )

View file

@ -17,7 +17,7 @@ func (k Kubernetes) records(state middleware.State, exact bool) ([]msg.Service,
if err != nil { if err != nil {
return nil, err return nil, err
} }
// TODO: Do we want to support the SkyDNS (hacky) Group feature? // TODO: Do we want to support the SkyDNS (hacky) Group feature?
services = msg.Group(services) services = msg.Group(services)
return services, nil return services, nil
} }
@ -141,7 +141,7 @@ func (k Kubernetes) AAAA(zone string, state middleware.State, previousRecords []
return records, nil return records, nil
} }
// SRV returns SRV records from etcd. // SRV returns SRV records from kubernetes.
// If the Target is not a name but an IP address, a name is created on the fly. // If the Target is not a name but an IP address, a name is created on the fly.
func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) {
services, err := k.records(state, false) services, err := k.records(state, false)
@ -208,13 +208,13 @@ func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR,
} }
// k.AAA(zone, state1, nil) as well...? // k.AAA(zone, state1, nil) as well...?
case ip.To4() != nil: case ip.To4() != nil:
serv.Host = k.Domain(serv.Key) serv.Host = serv.Key
srv := serv.NewSRV(state.QName(), weight) srv := serv.NewSRV(state.QName(), weight)
records = append(records, srv) records = append(records, srv)
extra = append(extra, serv.NewA(srv.Target, ip.To4())) extra = append(extra, serv.NewA(srv.Target, ip.To4()))
case ip.To4() == nil: case ip.To4() == nil:
serv.Host = k.Domain(serv.Key) serv.Host = serv.Key
srv := serv.NewSRV(state.QName(), weight) srv := serv.NewSRV(state.QName(), weight)
records = append(records, srv) records = append(records, srv)
@ -226,17 +226,17 @@ func (k Kubernetes) SRV(zone string, state middleware.State) (records []dns.RR,
// Returning MX records from kubernetes not implemented. // Returning MX records from kubernetes not implemented.
func (k Kubernetes) MX(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) { func (k Kubernetes) MX(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) {
return nil, nil, err return nil, nil, err
} }
// Returning CNAME records from kubernetes not implemented. // Returning CNAME records from kubernetes not implemented.
func (k Kubernetes) CNAME(zone string, state middleware.State) (records []dns.RR, err error) { func (k Kubernetes) CNAME(zone string, state middleware.State) (records []dns.RR, err error) {
return nil, err return nil, err
} }
// Returning TXT records from kubernetes not implemented. // Returning TXT records from kubernetes not implemented.
func (k Kubernetes) TXT(zone string, state middleware.State) (records []dns.RR, err error) { func (k Kubernetes) TXT(zone string, state middleware.State) (records []dns.RR, err error) {
return nil, err return nil, err
} }
func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dns.RR, err error) { func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dns.RR, err error) {
@ -259,11 +259,11 @@ func (k Kubernetes) NS(zone string, state middleware.State) (records, extra []dn
case ip == nil: case ip == nil:
return nil, nil, fmt.Errorf("NS record must be an IP address: %s", serv.Host) return nil, nil, fmt.Errorf("NS record must be an IP address: %s", serv.Host)
case ip.To4() != nil: case ip.To4() != nil:
serv.Host = k.Domain(serv.Key) serv.Host = serv.Key
records = append(records, serv.NewNS(state.QName())) records = append(records, serv.NewNS(state.QName()))
extra = append(extra, serv.NewA(serv.Host, ip.To4())) extra = append(extra, serv.NewA(serv.Host, ip.To4()))
case ip.To4() == nil: case ip.To4() == nil:
serv.Host = k.Domain(serv.Key) serv.Host = serv.Key
records = append(records, serv.NewNS(state.QName())) records = append(records, serv.NewNS(state.QName()))
extra = append(extra, serv.NewAAAA(serv.Host, ip.To16())) extra = append(extra, serv.NewAAAA(serv.Host, ip.To16()))
} }

View file

@ -0,0 +1,166 @@
package nametemplate
import (
"errors"
"fmt"
"strings"
"github.com/miekg/coredns/middleware/kubernetes/util"
)
// Likely symbols that require support:
// {id}
// {ip}
// {portname}
// {protocolname}
// {servicename}
// {namespace}
// {type} "svc" or "pod"
// {zone}
// SkyDNS normal services have an A-record of the form "{servicename}.{namespace}.{type}.{zone}"
// This resolves to the cluster IP of the service.
// SkyDNS headless services have an A-record of the form "{servicename}.{namespace}.{type}.{zone}"
// This resolves to the set of IPs of the pods selected by the Service. Clients are expected to
// consume the set or else use round-robin selection from the set.
var symbols = map[string]string{
"service": "{service}",
"namespace": "{namespace}",
"type": "{type}",
"zone": "{zone}",
}
var types = []string{
"svc",
"pod",
}
// TODO: Validate that provided NameTemplate string only contains:
// * valid, known symbols, or
// * static strings
// TODO: Support collapsing multiple segments into a symbol. Either:
// * all left-over segments are used as the "service" name, or
// * some scheme like "{namespace}.{namespace}" means use
// segments concatenated with a "." for the namespace, or
// * {namespace2:4} means use segements 2->4 for the namespace.
// TODO: possibly need to store length of segmented format to handle cases
// where query string segments to a shorter or longer list than the template.
// When query string segments to shorter than template:
// * either wildcards are being used, or
// * we are not looking up an A, AAAA, or SRV record (eg NS), or
// * we can just short-circuit failure before hitting the k8s API.
// Where the query string is longer than the template, need to define which
// symbol consumes the other segments. Most likely this would be the servicename.
// Also consider how to handle static strings in the format template.
type NameTemplate struct {
formatString string
splitFormat []string
// Element is a map of element name :: index in the segmented record name for the named element
Element map[string]int
}
func (t *NameTemplate) SetTemplate(s string) error {
var err error
fmt.Println()
t.Element = map[string]int{}
t.formatString = s
t.splitFormat = strings.Split(t.formatString, ".")
for templateIndex, v := range t.splitFormat {
elementPositionSet := false
for name, symbol := range symbols {
if v == symbol {
t.Element[name] = templateIndex
elementPositionSet = true
break
}
}
if !elementPositionSet {
if strings.Contains(v, "{") {
err = errors.New("Record name template contains the unknown symbol '" + v + "'")
fmt.Printf("[debug] %v\n", err)
return err
} else {
fmt.Printf("[debug] Template string has static element '%v'\n", v)
}
}
}
return err
}
// TODO: Find a better way to pull the data segments out of the
// query string based on the template. Perhaps it is better
// to treat the query string segments as a reverse stack and
// step down the stack to find the right element.
func (t *NameTemplate) GetZoneFromSegmentArray(segments []string) string {
if index, ok := t.Element["zone"]; !ok {
return ""
} else {
return strings.Join(segments[index:len(segments)], ".")
}
}
func (t *NameTemplate) GetNamespaceFromSegmentArray(segments []string) string {
return t.GetSymbolFromSegmentArray("namespace", segments)
}
func (t *NameTemplate) GetServiceFromSegmentArray(segments []string) string {
return t.GetSymbolFromSegmentArray("service", segments)
}
func (t *NameTemplate) GetTypeFromSegmentArray(segments []string) string {
typeSegment := t.GetSymbolFromSegmentArray("type", segments)
// Limit type to known types symbols
if util.StringInSlice(typeSegment, types) {
return ""
}
return typeSegment
}
func (t *NameTemplate) GetSymbolFromSegmentArray(symbol string, segments []string) string {
if index, ok := t.Element[symbol]; !ok {
return ""
} else {
return segments[index]
}
}
// GetRecordNameFromNameValues returns the string produced by applying the
// values to the NameTemplate format string.
func (t *NameTemplate) GetRecordNameFromNameValues(values NameValues) string {
recordName := make([]string, len(t.splitFormat))
copy(recordName[:], t.splitFormat)
for name, index := range t.Element {
if index == -1 {
continue
}
switch name {
case "type":
recordName[index] = values.TypeName
case "service":
recordName[index] = values.ServiceName
case "namespace":
recordName[index] = values.Namespace
case "zone":
recordName[index] = values.Zone
}
}
return strings.Join(recordName, ".")
}
type NameValues struct {
ServiceName string
Namespace string
TypeName string
Zone string
}

View file

@ -0,0 +1,129 @@
package nametemplate
import (
"fmt"
"strings"
"testing"
)
const (
zone = 0
namespace = 1
service = 2
)
// Map of format string :: expected locations of name symbols in the format.
// -1 value indicates that symbol does not exist in format.
var exampleTemplates = map[string][]int{
"{service}.{namespace}.{zone}": []int{2, 1, 0}, // service symbol expected @ position 0, namespace @ 1, zone @ 2
"{namespace}.{zone}": []int{1, 0, -1},
"": []int{-1, -1, -1},
}
func TestSetTemplate(t *testing.T) {
fmt.Printf("\n")
for s, expectedValue := range exampleTemplates {
n := new(NameTemplate)
n.SetTemplate(s)
// check the indexes resulting from calling SetTemplate() against expectedValues
if expectedValue[zone] != -1 {
if n.Element["zone"] != expectedValue[zone] {
t.Errorf("Expected zone at index '%v', instead found at index '%v' for format string '%v'", expectedValue[zone], n.Element["zone"], s)
}
}
}
}
func TestGetServiceFromSegmentArray(t *testing.T) {
var (
n *NameTemplate
formatString string
queryString string
splitQuery []string
expectedService string
actualService string
)
// Case where template contains {service}
n = new(NameTemplate)
formatString = "{service}.{namespace}.{zone}"
n.SetTemplate(formatString)
queryString = "myservice.mynamespace.coredns"
splitQuery = strings.Split(queryString, ".")
expectedService = "myservice"
actualService = n.GetServiceFromSegmentArray(splitQuery)
if actualService != expectedService {
t.Errorf("Expected service name '%v', instead got service name '%v' for query string '%v' and format '%v'", expectedService, actualService, queryString, formatString)
}
// Case where template does not contain {service}
n = new(NameTemplate)
formatString = "{namespace}.{zone}"
n.SetTemplate(formatString)
queryString = "mynamespace.coredns"
splitQuery = strings.Split(queryString, ".")
expectedService = ""
actualService = n.GetServiceFromSegmentArray(splitQuery)
if actualService != expectedService {
t.Errorf("Expected service name '%v', instead got service name '%v' for query string '%v' and format '%v'", expectedService, actualService, queryString, formatString)
}
}
func TestGetZoneFromSegmentArray(t *testing.T) {
var (
n *NameTemplate
formatString string
queryString string
splitQuery []string
expectedZone string
actualZone string
)
// Case where template contains {zone}
n = new(NameTemplate)
formatString = "{service}.{namespace}.{zone}"
n.SetTemplate(formatString)
queryString = "myservice.mynamespace.coredns"
splitQuery = strings.Split(queryString, ".")
expectedZone = "coredns"
actualZone = n.GetZoneFromSegmentArray(splitQuery)
if actualZone != expectedZone {
t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString)
}
// Case where template does not contain {zone}
n = new(NameTemplate)
formatString = "{service}.{namespace}"
n.SetTemplate(formatString)
queryString = "mynamespace.coredns"
splitQuery = strings.Split(queryString, ".")
expectedZone = ""
actualZone = n.GetZoneFromSegmentArray(splitQuery)
if actualZone != expectedZone {
t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString)
}
// Case where zone is multiple segments
n = new(NameTemplate)
formatString = "{service}.{namespace}.{zone}"
n.SetTemplate(formatString)
queryString = "myservice.mynamespace.coredns.cluster.local"
splitQuery = strings.Split(queryString, ".")
expectedZone = "coredns.cluster.local"
actualZone = n.GetZoneFromSegmentArray(splitQuery)
if actualZone != expectedZone {
t.Errorf("Expected zone name '%v', instead got zone name '%v' for query string '%v' and format '%v'", expectedZone, actualZone, queryString, formatString)
}
}

View file

@ -1,17 +0,0 @@
package kubernetes
import (
"strings"
"github.com/miekg/dns"
)
// Domain is the opposite of Path.
func (k Kubernetes) Domain(s string) string {
l := strings.Split(s, "/")
// start with 1, to strip /skydns
for i, j := 1, len(l)-1; i < j; i, j = i+1, j-1 {
l[i], l[j] = l[j], l[i]
}
return dns.Fqdn(strings.Join(l[1:len(l)-1], "."))
}

View file

@ -0,0 +1,48 @@
package kubernetes
import (
"fmt"
"github.com/miekg/dns"
)
// NormalizeZoneList filters the zones argument to remove
// array items that conflict with other items in zones.
// For example, providing the following zones array:
// [ "a.b.c", "b.c", "a", "e.d.f", "a.b" ]
// Returns:
// [ "a.b.c", "a", "e.d.f", "a.b" ]
// Zones filted out:
// - "b.c" because "a.b.c" and "b.c" share the common top
// level "b.c". First listed zone wins if there is a conflict.
//
// Note: This may prove to be too restrictive in practice.
// Need to find counter-example use-cases.
func NormalizeZoneList(zones []string) []string {
filteredZones := []string{}
for _, z := range zones {
zoneConflict, _ := subzoneConflict(filteredZones, z)
if zoneConflict {
fmt.Printf("[WARN] new zone '%v' from Corefile conflicts with existing zones: %v\n Ignoring zone '%v'\n", z, filteredZones, z)
} else {
filteredZones = append(filteredZones, z)
}
}
return filteredZones
}
// subzoneConflict returns true if name is a child or parent zone of
// any element in zones. If conflicts exist, return the conflicting zones.
func subzoneConflict(zones []string, name string) (bool, []string) {
conflicts := []string{}
for _, z := range zones {
if dns.IsSubDomain(z, name) || dns.IsSubDomain(name, z) {
conflicts = append(conflicts, z)
}
}
return (len(conflicts) != 0), conflicts
}

View file

@ -0,0 +1,32 @@
package kubernetes
import (
"testing"
)
// List of configured zones to test against
var confZones = []string{
"a.b.c",
"d",
}
// Map of zonename :: expected boolean result
var examplesSubzoneConflict = map[string]bool{
"a.b.c": true, // conflicts with zone "a.b.c"
"b.c": true, // conflicts with zone "a.b.c"
"c": true, // conflicts with zone "a.b.c"
"e": false, // no conflict
"a.b.c.e": false, // no conflict
"a.b.c.d": true, // conflicts with zone "d"
"": false,
}
func TestsubzoneConflict(t *testing.T) {
for z, expected := range examplesSubzoneConflict {
actual, conflicts := subzoneConflict(confZones, z)
if actual != expected {
t.Errorf("Expected conflict result '%v' for example '%v'. Instead got '%v'. Conflicting zones are: %v", expected, z, actual, conflicts)
}
}
}

View file

@ -0,0 +1,12 @@
// Package kubernetes/util provides helper functions for the kubernetes middleware
package util
// StringInSlice check whether string a is a member of slice.
func StringInSlice(a string, slice []string) bool {
for _, b := range slice {
if b == a {
return true
}
}
return false
}

View file

@ -0,0 +1,33 @@
package util
import (
"testing"
)
type InSliceData struct {
Slice []string
String string
InSlice bool
}
// Test data for TestStringInSlice cases.
var testdataInSlice = []struct {
Slice []string
String string
ExpectedResult bool
}{
{[]string{"a", "b", "c"}, "a", true},
{[]string{"a", "b", "c"}, "d", false},
{[]string{"a", "b", "c"}, "", false},
{[]string{}, "a", false},
{[]string{}, "", false},
}
func TestStringInSlice(t *testing.T) {
for _, example := range testdataInSlice {
actualResult := StringInSlice(example.String, example.Slice)
if actualResult != example.ExpectedResult {
t.Errorf("Expected stringInSlice result '%v' for example string='%v', slice='%v'. Instead got result '%v'.", example.ExpectedResult, example.String, example.Slice, actualResult)
}
}
}